Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
236fe47e4a |
@@ -1,4 +1 @@
|
||||
vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// AssignerModule is the asker the assigner's compose asks name.
|
||||
const AssignerModule = "mesh-delivery-assigner"
|
||||
|
||||
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
|
||||
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
|
||||
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
|
||||
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
|
||||
// second ask builds nothing.
|
||||
type composeAsks struct {
|
||||
ctx context.Context
|
||||
asker buildrun.Asker
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
asks map[string]*composeAsk
|
||||
}
|
||||
|
||||
type composeAsk struct {
|
||||
id string
|
||||
cancel context.CancelFunc
|
||||
ran *assign.Ran
|
||||
}
|
||||
|
||||
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
|
||||
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
|
||||
}
|
||||
|
||||
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
|
||||
// A job its layout refuses ends at once, failed with why: never silent.
|
||||
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if k, ok := a.asks[key]; ok {
|
||||
return k.ran, k.id
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
|
||||
At: a.now(), Compose: &job}
|
||||
ctx, cancel := context.WithCancel(a.ctx)
|
||||
k := &composeAsk{id: ask.ID, cancel: cancel}
|
||||
a.asks[key] = k
|
||||
if err := a.asker.Put(ctx, ask); err != nil {
|
||||
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
|
||||
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
|
||||
cancel()
|
||||
return k.ran, k.id
|
||||
}
|
||||
go func() {
|
||||
step, err := a.asker.Wait(ctx, ask)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
ran := assign.RanOf(step)
|
||||
if err != nil {
|
||||
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
|
||||
}
|
||||
a.mu.Lock()
|
||||
k.ran = &ran
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}()
|
||||
return nil, k.id
|
||||
}
|
||||
|
||||
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
|
||||
func (a *composeAsks) sweep(seen map[string]bool) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for key, k := range a.asks {
|
||||
if !seen[key] {
|
||||
k.cancel()
|
||||
delete(a.asks, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
|
||||
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
|
||||
// asks. The keys it used are marked in seen.
|
||||
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
|
||||
waiting := func(words string) delivery.CompositionVerdict {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
|
||||
}
|
||||
head, err := assign.HeadJob(m, ch)
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
|
||||
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
|
||||
}
|
||||
var pending []string
|
||||
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
|
||||
seen[key] = true
|
||||
headRan, id := p.asks.get(key, head)
|
||||
if headRan == nil {
|
||||
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
|
||||
}
|
||||
var readers []assign.Reader
|
||||
var notJudged []string
|
||||
for _, module := range sortedNames(assign.ReaderModules) {
|
||||
repository := assign.ReaderModules[module]
|
||||
var commits []string
|
||||
member := false
|
||||
if h := ch.Members[repository]; h != "" {
|
||||
commits, member = []string{h}, true
|
||||
} else {
|
||||
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
|
||||
"commit on record, and the group holds no member of "+repository)
|
||||
continue
|
||||
}
|
||||
for _, commit := range commits {
|
||||
job, err := assign.ReaderJob(ch, module, repository, commit)
|
||||
if err != nil {
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
|
||||
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
|
||||
continue
|
||||
}
|
||||
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
|
||||
seen[rkey] = true
|
||||
ran, rid := p.asks.get(rkey, job)
|
||||
if ran == nil {
|
||||
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
|
||||
continue
|
||||
}
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
|
||||
}
|
||||
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
|
||||
v.Warnings = append(v.Warnings, notJudged...)
|
||||
return v
|
||||
}
|
||||
|
||||
func sortedNames(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
|
||||
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
|
||||
type holder struct {
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
}
|
||||
|
||||
func (h *holder) taken() []delivery.StepAsk {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), h.asks...)
|
||||
}
|
||||
|
||||
func (b *bed) hold(t *testing.T) *holder {
|
||||
t.Helper()
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
|
||||
h := &holder{}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
a, err := delivery.DecodeStepAsk(m.Data())
|
||||
if err != nil || a.Compose == nil {
|
||||
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
|
||||
m.Ack()
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.asks = append(h.asks, a)
|
||||
h.mu.Unlock()
|
||||
req, err := composer.Decode(a.Compose.Request)
|
||||
if err == nil {
|
||||
err = composer.Fill(&req, func(digest string) ([]byte, error) {
|
||||
key, _ := delivery.ManifestsKey(digest)
|
||||
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
man, err := delivery.DecodeManifest(e.Value())
|
||||
return []byte(man.JSON), err
|
||||
})
|
||||
}
|
||||
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else if v, err := composer.Run(req); err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else {
|
||||
vr, _ := json.Marshal(v)
|
||||
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
|
||||
}
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
t.Errorf("the step cannot be written: %v", err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
if _, err := steps.Put(b.ctx, key, raw); err != nil {
|
||||
t.Errorf("the step %s: %v", key, err)
|
||||
}
|
||||
m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(cc.Stop)
|
||||
return h
|
||||
}
|
||||
|
||||
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
|
||||
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
|
||||
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// head's composer was built with, and asks nothing more on a later pass of the same heads.
|
||||
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
running := strings.Repeat("3", 40)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
|
||||
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
b.start(t)
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
// No holder takes the asks yet: the composition waits, naming them.
|
||||
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
|
||||
}
|
||||
h := b.hold(t)
|
||||
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
|
||||
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14", v)
|
||||
}
|
||||
asks := h.taken()
|
||||
if len(asks) != 2 {
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
|
||||
}
|
||||
byRepo := map[string]delivery.StepAsk{}
|
||||
for _, a := range asks {
|
||||
byRepo[a.Compose.Fetch["repository"].(string)] = a
|
||||
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
|
||||
t.Errorf("ask = %+v", a)
|
||||
}
|
||||
}
|
||||
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
|
||||
t.Errorf("the head's ask = %+v", a.Compose)
|
||||
}
|
||||
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running controller's ask = %+v", a.Compose)
|
||||
}
|
||||
// A later pass of the same heads (woken by any change) asks nothing more.
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := len(h.taken()); n != 2 {
|
||||
t.Errorf("a later pass of the same heads asked again: %d asks", n)
|
||||
}
|
||||
}
|
||||
|
||||
// verdict waits for the composition under key to satisfy ok, and answers it.
|
||||
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
|
||||
t.Helper()
|
||||
var last delivery.CompositionVerdict
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
|
||||
return last
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
|
||||
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
|
||||
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
|
||||
// where they part. The assigner composes nothing and never writes these.
|
||||
|
||||
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
|
||||
// it was refused or cannot be read.
|
||||
type publishedRow struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Unread string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
|
||||
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
|
||||
// A value or key that does not read is named with why, never left out.
|
||||
func published(values map[string][]byte) map[string][]publishedRow {
|
||||
byNode := map[string]map[string][]byte{}
|
||||
for key, raw := range values {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
if byNode[node] == nil {
|
||||
byNode[node] = map[string][]byte{}
|
||||
}
|
||||
byNode[node][key] = raw
|
||||
}
|
||||
out := map[string][]publishedRow{}
|
||||
for node, keys := range byNode {
|
||||
var rows []publishedRow
|
||||
for key, raw := range keys {
|
||||
row := publishedRow{Key: key}
|
||||
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
row.Unread = err.Error()
|
||||
case isNode:
|
||||
row.Module = "node"
|
||||
default:
|
||||
row.Module, row.FP = module, fp
|
||||
}
|
||||
if err == nil {
|
||||
c, derr := delivery.DecodeComposed(raw)
|
||||
switch {
|
||||
case derr != nil:
|
||||
row.Unread = derr.Error()
|
||||
case c.Refused != "":
|
||||
row.Refused = c.Refused
|
||||
default:
|
||||
if d, err := delivery.ComposedDigestOf(c); err != nil {
|
||||
row.Unread = err.Error()
|
||||
} else {
|
||||
row.Digest = d
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
|
||||
out[node] = rows
|
||||
}
|
||||
return out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,29 +14,23 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv(process.BrokerFileEnv) == "" {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
@@ -56,59 +50,37 @@ func main() {
|
||||
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
delivery.PartGrantsBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
}
|
||||
|
||||
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
|
||||
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
|
||||
var readFamilies = []string{delivery.ComposedRead}
|
||||
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
|
||||
// (ADR 0567 §1), named after the assigner as every module state is.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket,
|
||||
engineReadsBucket}
|
||||
|
||||
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
|
||||
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
|
||||
const engineReadsState = "engine-reads"
|
||||
|
||||
// engineReadsBucket is that state's bucket.
|
||||
const engineReadsBucket = delivery.AssignerModule + "_" + engineReadsState
|
||||
|
||||
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
|
||||
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
|
||||
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket}
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
|
||||
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
|
||||
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||
var keptStates = []announce.State{
|
||||
{Name: delivery.PlacementState, Readable: true},
|
||||
{Name: delivery.PlacementProposalsState, Readable: true},
|
||||
{Name: delivery.PlacementCandidatesState, Readable: true},
|
||||
{Name: delivery.PlacementPulsesState, Readable: true},
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
{Name: health.FindingsState, Readable: true},
|
||||
{Name: delivery.AssignerPoliciesState, Readable: true},
|
||||
{Name: delivery.SeatHoldersState, Readable: true},
|
||||
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
|
||||
{Name: views.StatusState, Readable: true}, // its first views of the indexer's states (views.go)
|
||||
{Name: engineReadsState, Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
@@ -126,52 +98,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.SeatHoldersBucket && err != nil {
|
||||
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
|
||||
// holds which seat here, and the controller's seats still answer.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == provisions.ProvisionsBucket && err != nil {
|
||||
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
|
||||
// until then no want waits for a provision, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.PoliciesBucket && err != nil {
|
||||
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
|
||||
// person's policy is read, so the controller's beside each composition holds, as before.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == assign.FindingsBucket && err != nil {
|
||||
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
|
||||
// router and kept as no finding.
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
|
||||
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
|
||||
// a record module is asked of a person, as before.
|
||||
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
|
||||
continue
|
||||
}
|
||||
if name == engineReadsBucket && err != nil {
|
||||
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
|
||||
// assigner_engine_reads lists nothing.
|
||||
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
}
|
||||
@@ -180,29 +106,36 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's
|
||||
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
|
||||
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, engineReadsBucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||
account, err := process.ReadBrokerAccount(getenv)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||
}
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, js, err := account.Dial()
|
||||
opts := []nats.Option{
|
||||
nats.Name(account.Node + "." + account.Module),
|
||||
nats.UserInfo(account.User, account.Password),
|
||||
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(account.URL, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -217,47 +150,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := said.Keeps(keptStates...); err != nil {
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
thisNode = account.Node
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
findings := findingsOf(ctx, kvs, logger.Printf)
|
||||
// Each change of the states its readers read is said on its views family (ADR 0615 §5, ADR 0619 §2); a watch of it
|
||||
// that ends stops the program, as a watch of what a pass reads does.
|
||||
viewErrs := make(chan error, len(viewedStates))
|
||||
startViews(ctx, js, conn, account, kvs, findings, viewErrs, logger.Printf)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case err := <-viewErrs:
|
||||
logger.Printf("%v: the assigner stops, and is started again", err)
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
iv := &indexerViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf,
|
||||
status: process.AwaitBucket(ctx, js, conn, firstViewsBucket, "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)}
|
||||
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, logger)
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
}
|
||||
|
||||
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
|
||||
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
|
||||
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
|
||||
var f assign.Findings
|
||||
if kv := kvs[assign.FindingsBucket]; kv != nil {
|
||||
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
|
||||
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
|
||||
f.Keys = func() ([]string, error) {
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
|
||||
var thisNode string
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
@@ -266,29 +162,24 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
delivery.PartGrantsBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.PlacementBucket,
|
||||
delivery.PlacementProposalsBucket,
|
||||
delivery.PlacementCandidatesBucket,
|
||||
delivery.PlacementPulsesBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.AssignerAsksBucket,
|
||||
delivery.AssignerCompositionsBucket,
|
||||
delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerWantsBucket,
|
||||
assign.PoliciesBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SeatHoldersBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
engineReadsBucket,
|
||||
}
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -299,16 +190,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
errs := make(chan error, len(watched)+2)
|
||||
settled := make(chan struct{}, len(watched)+2)
|
||||
// What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read
|
||||
// from the node's record where the engine does not answer (the fallback, said in the journal).
|
||||
reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf}
|
||||
if bus != nil {
|
||||
reads.Call = bus.CallOn
|
||||
}
|
||||
if kv := kvs[engineReadsBucket]; kv != nil {
|
||||
reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
reads.Delete = func(node string) error { return kv.Delete(ctx, node) }
|
||||
}
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
if err != nil {
|
||||
@@ -342,11 +223,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
if name == engineReadsBucket {
|
||||
reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed
|
||||
continue
|
||||
}
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
w.put(name, e.Key(), value)
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -386,57 +263,19 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && startsWithout(name) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket || name == delivery.SourceBucket {
|
||||
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
|
||||
opts = append(opts, jetstream.IncludeHistory())
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
}
|
||||
if err := watch(name, opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
|
||||
// (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands.
|
||||
if iv != nil {
|
||||
started += iv.start(ctx, w, wake, settled)
|
||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
|
||||
// module is placed.
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
placementAppeared := make(chan appeared, len(seatBuckets)+2)
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] == nil && js != nil {
|
||||
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
|
||||
}
|
||||
}
|
||||
takePlacement := func(a appeared) error {
|
||||
if kvs[a.name] != nil {
|
||||
return nil // bound already, by the other of its two events
|
||||
}
|
||||
kvs[a.name] = a.kv
|
||||
logger.Printf("the bucket %s is open and watched", a.name)
|
||||
return watch(a.name)
|
||||
}
|
||||
for range started {
|
||||
for range len(watched) + 1 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -445,23 +284,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
reads: reads}
|
||||
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
|
||||
// Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it
|
||||
// is not there yet, a node is asked again only when it comes back online.
|
||||
if js != nil {
|
||||
hearReadsChanged(ctx, js, reads, findings, logger.Printf)
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -470,29 +297,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
case err := <-errs:
|
||||
return err
|
||||
case <-settled:
|
||||
case kv := <-placementAppeared:
|
||||
if err := takePlacement(kv); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
|
||||
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
|
||||
// bus now holds, and says the seat waits while one is missing.
|
||||
var unbound []string
|
||||
for _, name := range seatBuckets {
|
||||
if kvs[name] != nil || js == nil {
|
||||
continue
|
||||
}
|
||||
if kv, err := js.KeyValue(ctx, name); err == nil {
|
||||
if err := takePlacement(appeared{name, kv}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
unbound = append(unbound, name)
|
||||
}
|
||||
p.sayWaits(unbound)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
@@ -503,13 +309,13 @@ var PutsReopenEvery = time.Minute
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) error {
|
||||
return func(c oc.Condition) error {
|
||||
subject, err := occ.ConditionSubject(delivery.AssignerModule)
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(delivery.AssignerModule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := occ.EncodeCondition(c)
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -517,39 +323,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's durable consumer on EVENTS,
|
||||
// and asks the node it names again. Any other event the consumer hands is acknowledged and left.
|
||||
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, reads *enginereads.Keeper, findings assign.Findings, logf func(string, ...any)) {
|
||||
account := process.Account{Node: thisNode, Module: delivery.AssignerModule}
|
||||
w := notHeardWriter(findings, logf)
|
||||
cons, err := account.EventConsumer(ctx, js, process.EventsStream)
|
||||
if err != nil {
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again (a node is asked again when it comes back online)",
|
||||
fmt.Errorf("the consumer %s on %s does not exist yet: %w", account.EventConsumerName(), process.EventsStream, err))
|
||||
return
|
||||
}
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
switch {
|
||||
case !ours:
|
||||
case err != nil:
|
||||
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
|
||||
reads.AskNode(node, enginereads.Event+" that does not read")
|
||||
default:
|
||||
reads.AskNode(node, enginereads.Event)
|
||||
}
|
||||
}, logf)
|
||||
if err != nil {
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again", err)
|
||||
return
|
||||
}
|
||||
w.Heard(process.EventsStream)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -18,44 +16,26 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
|
||||
findings := assign.Findings{Put: func(string, []byte) error { return nil }, Delete: func(string) error { return nil }}
|
||||
if viewsEmitter(nil, process.Account{}, nil, findings, nil, t.Logf).Findings == nil {
|
||||
t.Error("the assigner's views emitter writes no view-unsaid finding")
|
||||
}
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reads []string
|
||||
for _, r := range m.Reads {
|
||||
if slices.Contains(readFamilies, r) {
|
||||
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
|
||||
}
|
||||
reads = append(reads, strings.Replace(r, ".", "_", 1))
|
||||
}
|
||||
for _, l := range readFamilies {
|
||||
if !slices.Contains(m.Reads, l) {
|
||||
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
for _, s := range m.State {
|
||||
states = append(states, m.Module+"_"+s.Name)
|
||||
@@ -63,7 +43,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, pair := range []struct {
|
||||
what string
|
||||
code, want []string
|
||||
}{{"reads", readBuckets, reads}, {"states", append(slices.Clone(ownBuckets), marksBucket, firstViewsBucket), states}} { // the marks are bound by the views family (views.go)
|
||||
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
|
||||
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
|
||||
@@ -73,7 +53,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, s := range keptStates {
|
||||
kept = append(kept, m.Module+"_"+s.Name)
|
||||
}
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(append(slices.Clone(ownBuckets), marksBucket, firstViewsBucket))); !reflect.DeepEqual(got, want) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
@@ -82,19 +62,16 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
@@ -109,17 +86,15 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[delivery.PlacementBucket].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
@@ -130,6 +105,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
func TestNoCredential(t *testing.T) {
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
@@ -187,5 +170,3 @@ type kvEntry struct {
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -29,18 +25,11 @@ type passer struct {
|
||||
sdk string
|
||||
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(oc.Condition) error
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
|
||||
failedProvisions map[string]bool
|
||||
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
|
||||
missing, missingSaid map[string]bool
|
||||
// unplaced are the nodes whose unplaced finding this process wrote.
|
||||
unplaced map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
@@ -51,59 +40,6 @@ type passer struct {
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
|
||||
// provider is down, which holds no want): said once per words, and once more when it is reached again.
|
||||
unreached map[string]string
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
|
||||
// (clearStaleCalls): the assigner makes no such call now (issue 774).
|
||||
callsCleared bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// convertedAll says every node in the form before was converted to its node and module keys (convertPlacement).
|
||||
convertedAll bool
|
||||
// saidWaits is the last said of the seat's wait for its states (sayWaits).
|
||||
saidWaits string
|
||||
intended map[string]map[string]map[string]bool
|
||||
// reads keeps what each node's engine states it reads, asked through the node-engine seat's verb (ADR 0618 §3); nil reads
|
||||
// each node's record, as the rules did before the verb. online is, by node, whether the last pass saw it online.
|
||||
reads *enginereads.Keeper
|
||||
online map[string]bool
|
||||
}
|
||||
|
||||
// engineReads sets what the rules read of each node's engine (ADR 0618 §3, §4): the statement the verb manifest-reads
|
||||
// answered, or its record where the verb was not answered (the fallback) or not asked yet. A node is asked when the
|
||||
// assigner first hears of it, and again when it comes back online, since an engine built before the event says none.
|
||||
func (p *passer) engineReads(n assign.Nodes) {
|
||||
if p.reads == nil {
|
||||
return
|
||||
}
|
||||
known := p.w.knownNodes()
|
||||
first := p.online == nil
|
||||
for _, node := range known {
|
||||
switch {
|
||||
case first:
|
||||
p.reads.AskNode(node, "the assigner started")
|
||||
case !p.reads.Known(node):
|
||||
p.reads.AskNode(node, "first heard of")
|
||||
case !first && n.Online[node] && !p.online[node]:
|
||||
p.reads.AskNode(node, "back online")
|
||||
}
|
||||
}
|
||||
p.online = maps.Clone(n.Online)
|
||||
if p.online == nil {
|
||||
p.online = map[string]bool{}
|
||||
}
|
||||
for node, reads := range p.reads.Reads(known) {
|
||||
if p.reads.Known(node) {
|
||||
n.Reads[node] = reads
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
@@ -114,27 +50,26 @@ func (p *passer) clock() time.Time {
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.placementSeat(ctx)
|
||||
p.compositions(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
|
||||
for node := range placed {
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
@@ -143,8 +78,8 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
|
||||
assigned := p.w.assigned()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
for name, mod := range trunk {
|
||||
@@ -156,7 +91,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
nodes := p.w.nodes(manifests, assigned)
|
||||
p.engineReads(nodes)
|
||||
nodesOf := map[string][]string{}
|
||||
for node, modules := range assigned {
|
||||
for _, m := range modules {
|
||||
@@ -164,14 +98,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
provisionsOf := p.provisionsOf()
|
||||
missing := map[string]bool{}
|
||||
defer func() { p.provisionsMissing(ctx, missing) }()
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
@@ -188,25 +114,15 @@ func (p *passer) wants(ctx context.Context) {
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
// The verb edges, once for the pass (ADR 0614 §2, §3).
|
||||
moduleGraph := p.moduleGraph()
|
||||
served := p.served(ctx, get, moduleGraph, nodes)
|
||||
offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5)
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
},
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.PartGrantKey(name, fp)
|
||||
return err == nil && p.w.has(grants, key)
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
@@ -215,17 +131,16 @@ func (p *passer) wants(ctx context.Context) {
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
},
|
||||
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
|
||||
}}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.PartGrantKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
@@ -233,26 +148,13 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
if served != nil {
|
||||
m.Invokes, m.Served = keptInvokes(moduleGraph, name), served
|
||||
}
|
||||
m.Offering = offering
|
||||
last := map[string]string{}
|
||||
for _, node := range m.Nodes {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
for _, node := range m.Nodes {
|
||||
for _, prov := range m.MissingProvisions(node) {
|
||||
missing[node+"."+name+"."+prov] = true
|
||||
}
|
||||
}
|
||||
p.yeses(ctx, name, m, wanted[name], &out)
|
||||
gatherNeither(neither, name, out)
|
||||
leftNeither(left, m, wanted[name], out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
p.sayUnreached(name, m.Nodes, out)
|
||||
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
@@ -308,7 +210,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = h.Reason()
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
@@ -327,80 +229,6 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
|
||||
}
|
||||
}
|
||||
|
||||
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
|
||||
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
|
||||
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
|
||||
if p.unreached == nil {
|
||||
p.unreached = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
key := node + "." + module
|
||||
why, last := out.Unreached[node], p.unreached[key]
|
||||
switch {
|
||||
case why != "" && why != last:
|
||||
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
|
||||
p.unreached[key] = why
|
||||
case why == "" && last != "":
|
||||
p.logf("%s on %s is reached: its providers run healthy", module, node)
|
||||
delete(p.unreached, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
|
||||
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
|
||||
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
|
||||
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
|
||||
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
|
||||
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
|
||||
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
|
||||
if p.nodeFailed == nil {
|
||||
p.nodeFailed = map[string]string{}
|
||||
}
|
||||
look := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
look[node] = true
|
||||
}
|
||||
for key := range p.nodeFailed {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
if first {
|
||||
for node := range known.Online {
|
||||
look[node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(look)) {
|
||||
key := node + "." + module
|
||||
f, failing := out.NodeFailed[node]
|
||||
last, said := p.nodeFailed[key]
|
||||
switch {
|
||||
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
|
||||
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
|
||||
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
|
||||
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
|
||||
case !failing && (said || first):
|
||||
if said {
|
||||
p.logf("%s on %s no longer fails its health check alone", module, node)
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
|
||||
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
delete(p.nodeFailed, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
@@ -414,48 +242,6 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
|
||||
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
|
||||
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
|
||||
// left it with none).
|
||||
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; on {
|
||||
continue
|
||||
}
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
why := ""
|
||||
h, ruled := out.Rules[node]
|
||||
switch {
|
||||
case written && !w.Removed:
|
||||
case (m.Held && !out.Lifted) || out.Held != nil:
|
||||
why = "it is held, and its held condition says so"
|
||||
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
|
||||
why = "its move waits for the operator's yes on its record ask"
|
||||
case putOn(m, node):
|
||||
why = "its put waits on its own steps and the operator's answer"
|
||||
case m.FP == "":
|
||||
why = "the trunk has no build of it"
|
||||
case ruled:
|
||||
why = h.Reason()
|
||||
default:
|
||||
why = "the node does not run it"
|
||||
}
|
||||
into[node+"."+m.Name] = why
|
||||
}
|
||||
}
|
||||
|
||||
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
|
||||
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
|
||||
func putOn(m assign.Module, node string) bool {
|
||||
p, put := m.Puts[node]
|
||||
_, owns := assign.PutOwns(p)
|
||||
return put && owns
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
@@ -472,9 +258,7 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
|
||||
// it does not hold is no longer assigned.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
@@ -494,33 +278,76 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]st
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Reason())
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
why, assigned := left[key]
|
||||
if !assigned {
|
||||
why = "it is no longer assigned there"
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if why == "" {
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
} else {
|
||||
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||
func (p *passer) compositions(ctx context.Context) {
|
||||
if p.sdk == "" {
|
||||
p.sdk = sdkversion.Version()
|
||||
}
|
||||
written := p.w.read(delivery.AssignerCompositionsBucket)
|
||||
trunk := p.w.trunk()
|
||||
var mesh *assign.Mesh
|
||||
seen := map[string]bool{}
|
||||
if p.asks != nil {
|
||||
defer p.asks.sweep(seen)
|
||||
}
|
||||
for key, value := range p.w.read(delivery.CandidatesBucket) {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err)
|
||||
continue
|
||||
}
|
||||
out, err := delivery.CompositionKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
p.logf("the candidate %s names no composition key: %v", key, err)
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk)
|
||||
var v delivery.CompositionVerdict
|
||||
if waits != "" {
|
||||
v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits}
|
||||
} else {
|
||||
if mesh == nil {
|
||||
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
|
||||
mesh = &m
|
||||
}
|
||||
if ch.SDKHead != "" && p.asks != nil {
|
||||
// A group with a mesh-sdk member is composed by the composer built at its head (ADR 0482 clauses 3 to 5).
|
||||
v = p.composeAtHead(ctx, *mesh, ch, seen, began)
|
||||
} else {
|
||||
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||
}
|
||||
}
|
||||
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||
before.Took, before.V = v.Took, 0
|
||||
if delivery.SameCompositionVerdict(before, v) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) })
|
||||
}
|
||||
}
|
||||
|
||||
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
|
||||
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
|
||||
value, err := encode()
|
||||
@@ -528,11 +355,6 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
p.logf("%s %s is not written: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
// A value the bucket already holds is not put again: every put is a change its watchers hear, and a want put unchanged
|
||||
// made its node compose again (issue 774, 2026-10-31: novox composed every 8 s, two asks of the vault each time).
|
||||
if held := p.w.value(bucket, key); held != nil && bytes.Equal(held, value) {
|
||||
return
|
||||
}
|
||||
if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be written: %v", bucket, key, err)
|
||||
return
|
||||
@@ -554,7 +376,7 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery
|
||||
asking := int(now.Unix())
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
||||
@@ -612,14 +434,6 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||
if !on {
|
||||
return true
|
||||
}
|
||||
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
|
||||
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
|
||||
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
|
||||
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||
@@ -737,7 +551,7 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
||||
}
|
||||
words := p.heldHold(module, fp, nil).Words
|
||||
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
||||
@@ -749,12 +563,12 @@ func (p *passer) answeredWith(key, option string) bool {
|
||||
if !ok || a.Option != option {
|
||||
return false
|
||||
}
|
||||
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||
return id == ask.ID
|
||||
return id == ask.Ask.ID
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
@@ -779,7 +593,7 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
|
||||
return
|
||||
}
|
||||
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
|
||||
@@ -810,7 +624,7 @@ func (p *passer) backups(ctx context.Context) {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
|
||||
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
|
||||
if there != nil || assign.BackupAskDigest(current) != want.Composed {
|
||||
after := -1
|
||||
if there == nil {
|
||||
@@ -820,7 +634,7 @@ func (p *passer) backups(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID ||
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
|
||||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
|
||||
continue // not answered yet: the ask stands
|
||||
}
|
||||
@@ -831,7 +645,7 @@ func (p *passer) backups(ctx context.Context) {
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
w := want
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
|
||||
p.answeredAt[currentAsk.ID] = haveChanges
|
||||
p.answeredAt[current.Ask.ID] = haveChanges
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -839,8 +653,8 @@ func (p *passer) backups(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
|
||||
if seen, known := p.answeredAt[currentAsk.ID]; !known {
|
||||
p.answeredAt[currentAsk.ID] = haveChanges // after a restart: wait for the next failure
|
||||
if seen, known := p.answeredAt[current.Ask.ID]; !known {
|
||||
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
|
||||
} else if haveChanges > seen {
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
|
||||
}
|
||||
@@ -850,7 +664,7 @@ func (p *passer) backups(ctx context.Context) {
|
||||
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
|
||||
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
|
||||
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
c := assign.BackupCondition(node, module, want.FP, words, true)
|
||||
if p.say == nil || p.said[c.Key] {
|
||||
return
|
||||
@@ -861,306 +675,3 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
|
||||
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
|
||||
// want naming the release, with no ask (ADR 0601 §1).
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if m.Self == nil || !m.Self(node) {
|
||||
continue
|
||||
}
|
||||
was, had := before[node]
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
if !had || was.Removed {
|
||||
continue
|
||||
}
|
||||
w = was
|
||||
}
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes, release := "", ""
|
||||
if had && was.FP == w.FP {
|
||||
yes, release = was.Yes, was.Release
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
switch {
|
||||
case m.Allowed != nil && m.Allowed(node, h.Waits):
|
||||
yes, release = h.Waits, ""
|
||||
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
|
||||
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
|
||||
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
|
||||
if r, ok := m.Release(w.FP); ok {
|
||||
yes, release = h.Waits, r
|
||||
break
|
||||
}
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
default:
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes == "" {
|
||||
release = ""
|
||||
}
|
||||
if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chosenOf is a person's choice for module, nil when none.
|
||||
func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy {
|
||||
if c, ok := chosen[module]; ok {
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
|
||||
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
|
||||
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
|
||||
// needs-operator), deleted once it is no longer failed.
|
||||
func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision {
|
||||
if p.kvs[provisions.ProvisionsBucket] == nil {
|
||||
return func(string) func(string) map[string]provisions.Provision { return nil }
|
||||
}
|
||||
by := map[string]map[string]map[string]provisions.Provision{} // module, node, key
|
||||
failed := map[string]provisions.Provision{}
|
||||
for key, raw := range p.w.read(provisions.ProvisionsBucket) {
|
||||
pr, err := provisions.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
node, module := pr.Consumer.Node, pr.Consumer.Module
|
||||
if by[module] == nil {
|
||||
by[module] = map[string]map[string]provisions.Provision{}
|
||||
}
|
||||
if by[module][node] == nil {
|
||||
by[module][node] = map[string]provisions.Provision{}
|
||||
}
|
||||
by[module][node][key] = pr
|
||||
if pr.State == provisions.Failed {
|
||||
failed[key] = pr
|
||||
}
|
||||
}
|
||||
p.provisionFindings(failed)
|
||||
return func(module string) func(string) map[string]provisions.Provision {
|
||||
return func(node string) map[string]provisions.Provision { return by[module][node] }
|
||||
}
|
||||
}
|
||||
|
||||
// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed.
|
||||
func (p *passer) provisionFindings(failed map[string]provisions.Provision) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now
|
||||
if p.now != nil {
|
||||
now = p.now
|
||||
}
|
||||
for key, pr := range failed {
|
||||
if p.failedProvisions[key] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionFailedFinding(key, pr, now())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionFailedKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the failed provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.failedProvisions == nil {
|
||||
p.failedProvisions = map[string]bool{}
|
||||
}
|
||||
p.failedProvisions[key] = true
|
||||
}
|
||||
for key := range p.failedProvisions {
|
||||
if _, still := failed[key]; still {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil {
|
||||
p.logf("the failed provision %s is not cleared: %v", key, err)
|
||||
continue
|
||||
}
|
||||
delete(p.failedProvisions, key)
|
||||
}
|
||||
}
|
||||
|
||||
// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
|
||||
// when its manifest names none (ADR 0564 clause 13 (d)).
|
||||
var MissingBound = delivery.DefaultDeliveryBound
|
||||
|
||||
// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
|
||||
func missingPulse(key string) string { return "provision-missing." + key }
|
||||
|
||||
// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
|
||||
// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
|
||||
// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
|
||||
// the watch, is the pass that writes it: no timer.
|
||||
func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
|
||||
pulses := p.kvs[delivery.PlacementPulsesBucket]
|
||||
if p.missing == nil {
|
||||
p.missing = map[string]bool{}
|
||||
}
|
||||
for key := range missing {
|
||||
switch {
|
||||
case !p.missing[key]:
|
||||
p.missing[key] = true
|
||||
if pulses != nil {
|
||||
p.pulse(ctx, missingPulse(key), MissingBound)
|
||||
}
|
||||
case pulses == nil || !p.pulsing(missingPulse(key)):
|
||||
if p.missingSaid[key] || p.findings.Put == nil {
|
||||
continue
|
||||
}
|
||||
value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.ProvisionMissingKey(key), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the missing provision %s is not said: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if p.missingSaid == nil {
|
||||
p.missingSaid = map[string]bool{}
|
||||
}
|
||||
p.missingSaid[key] = true
|
||||
}
|
||||
}
|
||||
for key := range p.missing {
|
||||
if missing[key] {
|
||||
continue
|
||||
}
|
||||
delete(p.missing, key)
|
||||
if pulses != nil && p.pulsing(missingPulse(key)) {
|
||||
if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil)
|
||||
}
|
||||
}
|
||||
if p.missingSaid[key] && p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
|
||||
p.logf("the missing provision %s is not cleared: %v", key, err)
|
||||
p.missing[key] = true
|
||||
continue
|
||||
}
|
||||
delete(p.missingSaid, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unplacedFindings writes, from the lease write, the finding `machine.<node>.unplaced` for each node that runs but no
|
||||
// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed.
|
||||
func (p *passer) unplacedFindings() {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := []string(p.w.unplacedNodes())
|
||||
is := map[string]bool{}
|
||||
for _, node := range now {
|
||||
is[node] = true
|
||||
if p.unplaced[node] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.UnplacedFinding(node, p.clock())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.UnplacedKey(node), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the unplaced node %s is not said: %v", node, err)
|
||||
continue
|
||||
}
|
||||
if p.unplaced == nil {
|
||||
p.unplaced = map[string]bool{}
|
||||
}
|
||||
p.unplaced[node] = true
|
||||
}
|
||||
for node := range p.unplaced {
|
||||
if is[node] {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil {
|
||||
delete(p.unplaced, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per
|
||||
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
|
||||
// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no
|
||||
// timer writes it.
|
||||
func (p *passer) seatHolders(ctx context.Context) {
|
||||
kv := p.kvs[delivery.SeatHoldersBucket]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
placements := p.w.placements()
|
||||
if len(placements) == 0 {
|
||||
return // no placement read yet: nothing is known of who holds what, and nothing is deleted
|
||||
}
|
||||
want := delivery.SeatHoldersOf(placements)
|
||||
have := p.w.read(delivery.SeatHoldersBucket)
|
||||
for key, sh := range want {
|
||||
value, err := json.Marshal(sh)
|
||||
if err != nil || string(have[key]) == string(value) {
|
||||
continue
|
||||
}
|
||||
p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil })
|
||||
}
|
||||
for key := range have {
|
||||
if _, ok := want[key]; ok {
|
||||
continue
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil {
|
||||
p.logf("the seat holders %s are not deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.SeatHoldersBucket, key, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name
|
||||
// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from.
|
||||
func offered(manifests map[string]catalogue.Manifest) func(string) bool {
|
||||
names := map[string]bool{}
|
||||
add := func(m catalogue.Manifest) {
|
||||
for _, o := range m.Offers() {
|
||||
names[o] = true
|
||||
}
|
||||
}
|
||||
for _, m := range manifests {
|
||||
add(m)
|
||||
}
|
||||
for _, module := range delivery.ProvidedModules() {
|
||||
if m, _, err := delivery.ProvidedManifest(module); err == nil {
|
||||
add(m)
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return names[provision] }
|
||||
}
|
||||
|
||||
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
|
||||
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
|
||||
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
|
||||
return func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) {
|
||||
if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -13,15 +12,10 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -30,16 +24,9 @@ import (
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
puts int // how many puts reached the bucket
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
m.puts++
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
@@ -55,34 +42,6 @@ func newPasser(kvs buckets, at time.Time) *passer {
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
|
||||
// (research 072 row 37).
|
||||
func placeOn(t *testing.T, w *world, node string, modules ...string) {
|
||||
t.Helper()
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
|
||||
func composes(t *testing.T, w *world, node, digest string) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
@@ -95,7 +54,7 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
||||
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
||||
first.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a1, err := decodeAsk(key, asksKV.values[key])
|
||||
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -105,7 +64,7 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
}
|
||||
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
||||
again.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a2, err := decodeAsk(key, asksKV.values[key])
|
||||
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -117,24 +76,16 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
// ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides
|
||||
// itself (mesh-addressing, mesh-wireguard's); nothing else.
|
||||
off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}})
|
||||
for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} {
|
||||
if off(provision) != want {
|
||||
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
|
||||
}
|
||||
}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
||||
v, err := assign.EncodeAsk(key, a)
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
return mustAskID(a)
|
||||
return a.Ask.ID
|
||||
}
|
||||
first := put(100)
|
||||
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
||||
@@ -154,89 +105,6 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
// ADR 0614 §4: the rules read what each node's engine states it reads from its record in `<node>.node`; a node with no
|
||||
// record, or one that states nothing, is read as stating nothing (the rules then read the baseline).
|
||||
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
n := w.nodes(nil, nil)
|
||||
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
|
||||
t.Errorf("the nodes' reads are %v", n.Reads)
|
||||
}
|
||||
// ADR 0618 §3: with the verb asked, a node's reads are what its engine answered; where the engine answers it does not
|
||||
// serve the verb, its record is read once in its place (the fallback, issue 774); a node asked again when it comes
|
||||
// back online, since an engine built before the event says none.
|
||||
asked := map[string]int{}
|
||||
call := func(node, key string, _ any) (json.RawMessage, error) {
|
||||
asked[node]++
|
||||
if node == "g14" || key != enginereads.Invoke {
|
||||
return nil, &busretry.Answered{Words: "the seat node-engine serves no verb manifest-reads"}
|
||||
}
|
||||
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
|
||||
}
|
||||
p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}}
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: false})
|
||||
w.put(delivery.PresenceBucket, "ace", on)
|
||||
n = w.nodes(nil, nil)
|
||||
p.engineReads(n)
|
||||
if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 {
|
||||
t.Errorf("with the verb the nodes' reads are %v (asked %v)", n.Reads, asked)
|
||||
}
|
||||
if s := p.reads.Statements(); s["ace"].From != enginereads.FromVerb || s["g14"].From != enginereads.FromRecord {
|
||||
t.Errorf("the statements are %+v", s)
|
||||
}
|
||||
p.engineReads(w.nodes(nil, nil))
|
||||
on, _ = delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
w.put(delivery.PresenceBucket, "ace", on)
|
||||
p.engineReads(w.nodes(nil, nil))
|
||||
if asked["ace"] != 2 {
|
||||
t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"])
|
||||
}
|
||||
// The contract carries no checks, so its reader judges what the engine says (mesh-contracts seats/nodeengine): an
|
||||
// answer or an event that does not hold is no statement, and an event's node is its subject's last token.
|
||||
now := time.Now().Format(time.RFC3339Nano)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
raw string
|
||||
stays bool
|
||||
}{
|
||||
{"a whole answer", `{"v":1,"node":"ace","manifest-reads":["form:new"],"stated-at":"` + now + `"}`, true},
|
||||
{"an engine stating nothing", `{"v":1,"node":"ace","stated-at":"` + now + `"}`, true},
|
||||
{"another layout", `{"v":2,"node":"ace","stated-at":"` + now + `"}`, false},
|
||||
{"no node", `{"v":1,"node":"","stated-at":"` + now + `"}`, false},
|
||||
{"a node that is no token", `{"v":1,"node":"a.ce","stated-at":"` + now + `"}`, false},
|
||||
{"another node's answer", `{"v":1,"node":"g14","stated-at":"` + now + `"}`, false},
|
||||
{"an empty capability", `{"v":1,"node":"ace","manifest-reads":[""],"stated-at":"` + now + `"}`, false},
|
||||
{"a time that is no time", `{"v":1,"node":"ace","stated-at":"yesterday"}`, false},
|
||||
} {
|
||||
s := enginereads.Ask(func(string, string, any) (json.RawMessage, error) { return json.RawMessage(row.raw), nil }, "ace", time.Now())
|
||||
if (s.From == enginereads.FromVerb) != row.stays || !s.Answered {
|
||||
t.Errorf("answer %s: %+v", row.name, s)
|
||||
}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name, subject, raw string
|
||||
ours, fails bool
|
||||
}{
|
||||
{"the event", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"` + now + `","why":"returned"}`, true, false},
|
||||
{"another reason", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"` + now + `","why":"bored"}`, true, true},
|
||||
{"another node in the body", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"g14","at":"` + now + `","why":"started"}`, true, true},
|
||||
{"a time that is no time", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"x","why":"started"}`, true, true},
|
||||
{"another event", "mesh.seat.node-engine.event.done.ace", `{}`, false, false},
|
||||
} {
|
||||
_, ours, err := enginereads.NodeOf(row.subject, []byte(row.raw))
|
||||
if ours != row.ours || (err != nil) != row.fails {
|
||||
t.Errorf("event %s: ours %v, %v", row.name, ours, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
@@ -267,9 +135,6 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
if s[state] == nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
@@ -333,7 +198,7 @@ func putOfTheBus(t *testing.T) {
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "nats")
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
@@ -389,17 +254,19 @@ func putOfTheBus(t *testing.T) {
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
composes(t, p.w, "novox", digest)
|
||||
gkey, _ := delivery.PartGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
world(delivery.PartGrantsBucket, gkey, g)
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||
}
|
||||
a, err := decodeAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != occ.Approve {
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||
}
|
||||
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||
@@ -417,7 +284,7 @@ func putOfTheBus(t *testing.T) {
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
|
||||
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
@@ -447,7 +314,7 @@ func putOfTheBus(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := step()
|
||||
a2, err := decodeAsk(akey, asksKV.values[akey])
|
||||
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||
}
|
||||
@@ -475,15 +342,10 @@ func putOfTheBus(t *testing.T) {
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
|
||||
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []oc.Condition
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
p.say = func(c oc.Condition) error {
|
||||
p.say = func(c asks.Condition) error {
|
||||
if failing {
|
||||
failing = false
|
||||
return errors.New("the channel is away")
|
||||
@@ -495,67 +357,30 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil, nil) // the say fails
|
||||
p.sayNeither(one, nil, nil)
|
||||
p.sayNeither(one, nil, nil)
|
||||
if len(said) != 1 || said[0].State != occ.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
wants := map[string]string{"g14.screen-lock": ""}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
p.sayNeither(map[string]neitherOne{}, nil, wants)
|
||||
if len(said) != 2 || said[1].State != occ.StateCleared || said[1].Key != said[0].Key {
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
|
||||
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: delivery.RuleGrant, Words: "no credential"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != occ.StateRaised || !strings.HasPrefix(c.Explanation, occ.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
if c := said[len(said)-1]; c.State != occ.StateCleared || strings.Contains(c.Resolved, "has a want") {
|
||||
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
|
||||
}
|
||||
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
|
||||
t.Errorf("the clear without a want said %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
|
||||
// through its steps owns the want, so it is named, never "the node does not run it".
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
puts map[string]delivery.Put
|
||||
before map[string]delivery.Want
|
||||
why string
|
||||
}{
|
||||
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
|
||||
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
|
||||
{"nothing", nil, nil, "the node does not run it"},
|
||||
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
|
||||
} {
|
||||
left := map[string]string{}
|
||||
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
|
||||
if why, there := left["novox.nats"]; !there || why != c.why {
|
||||
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
|
||||
}
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
@@ -581,78 +406,19 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
|
||||
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
|
||||
var conds []oc.Condition
|
||||
p.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
|
||||
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
|
||||
for i, c := range []struct {
|
||||
failed map[string]assign.NodeFailure
|
||||
state string // the condition said in this pass, "" for none
|
||||
}{
|
||||
{failing, occ.StateRaised},
|
||||
{failing, ""},
|
||||
{nil, occ.StateCleared},
|
||||
{nil, ""},
|
||||
} {
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
|
||||
switch {
|
||||
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
|
||||
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
|
||||
case c.state == occ.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
|
||||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
|
||||
t.Errorf("node failed, raised %+v", conds[0])
|
||||
}
|
||||
}
|
||||
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
|
||||
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
|
||||
conds = nil
|
||||
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
|
||||
if len(conds) != 2 || conds[1].State != occ.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
|
||||
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
|
||||
}
|
||||
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
|
||||
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
known assign.Nodes
|
||||
keys []string
|
||||
}{
|
||||
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
|
||||
} {
|
||||
conds = nil
|
||||
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
|
||||
fresh.logf = func(string, ...any) {}
|
||||
fresh.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
|
||||
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
|
||||
var keys []string
|
||||
for _, cond := range conds {
|
||||
if cond.State == occ.StateCleared {
|
||||
keys = append(keys, cond.Key)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
|
||||
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []oc.Condition
|
||||
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -663,7 +429,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
p.wants(context.Background())
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range said {
|
||||
if c.State != occ.StateCleared {
|
||||
if c.State != asks.StateCleared {
|
||||
t.Errorf("the first pass raised %+v", c)
|
||||
}
|
||||
cleared[c.Key] = true
|
||||
@@ -683,14 +449,14 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]oc.Condition) {
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]oc.Condition{}
|
||||
p.say = func(c oc.Condition) error { *said = append(*said, c); return nil }
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
placeOn(t, p.w, "shanks", "systemd-resolved")
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
@@ -701,12 +467,12 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
return p, said
|
||||
}
|
||||
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
|
||||
neithers := func(said []oc.Condition) (raised, cleared int) {
|
||||
neithers := func(said []asks.Condition) (raised, cleared int) {
|
||||
for _, c := range said {
|
||||
if c.Key != key {
|
||||
continue
|
||||
}
|
||||
if c.State == occ.StateRaised {
|
||||
if c.State == asks.StateRaised {
|
||||
raised++
|
||||
} else {
|
||||
cleared++
|
||||
@@ -754,19 +520,21 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the node's own composition is not read
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
|
||||
{"the trunk is not read at all", nil, nil, false},
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []oc.Condition
|
||||
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
for _, key := range all {
|
||||
k, _ := delivery.ParseAnyAskKey(key)
|
||||
var a delivery.AssignerAsk
|
||||
@@ -780,27 +548,39 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
default:
|
||||
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
|
||||
}
|
||||
v, err := assign.EncodeAsk(key, a)
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
|
||||
if !c.noDigit {
|
||||
composes(t, p.w, "novox", digNew)
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
|
||||
firstAsk, _ := decodeAsk(holdOld, asksKV.values[holdOld])
|
||||
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
p.staleAsks(context.Background(), c.trunk)
|
||||
// the held record and condition of a fingerprint whose hold ask went are taken away with it
|
||||
heldOld, _ = delivery.HeldKey("web", fpOld)
|
||||
_, heldThere := wantsKV.values[heldOld]
|
||||
cleared := slices.ContainsFunc(said, func(c oc.Condition) bool {
|
||||
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == occ.StateCleared
|
||||
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
|
||||
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
|
||||
})
|
||||
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
|
||||
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
|
||||
@@ -817,8 +597,8 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
p.now = func() time.Time { return time.Now().Add(time.Hour) }
|
||||
p.holdAsk(context.Background(), "web", fpOld, nil)
|
||||
p.heldCondition(context.Background(), "web", fpOld, nil, false)
|
||||
again, err := decodeAsk(holdOld, asksKV.values[holdOld])
|
||||
raised := slices.ContainsFunc(said, func(c oc.Condition) bool { return c.State == occ.StateRaised })
|
||||
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
|
||||
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
|
||||
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
|
||||
}
|
||||
@@ -829,9 +609,9 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []oc.Condition
|
||||
var said []asks.Condition
|
||||
fail := true
|
||||
p.say = func(c oc.Condition) error {
|
||||
p.say = func(c asks.Condition) error {
|
||||
if fail {
|
||||
fail = false
|
||||
return errors.New("router unreachable")
|
||||
@@ -840,7 +620,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
|
||||
v, err := assign.EncodeAsk(holdOld, a)
|
||||
v, err := delivery.EncodeAssignerAsk(holdOld, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -851,7 +631,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
|
||||
}
|
||||
p.staleAsks(context.Background(), trunkOf(fpNew))
|
||||
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != occ.StateCleared {
|
||||
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
|
||||
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
|
||||
}
|
||||
})
|
||||
@@ -885,290 +665,51 @@ func mustKey(key string, err error) string {
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
aReleasesApproveAnswersTheMoveOfItsBuildAlone(t)
|
||||
}
|
||||
|
||||
// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is
|
||||
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
|
||||
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
|
||||
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
commit := strings.Repeat("1", 40)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
alone bool
|
||||
at string // the fingerprint the have runs
|
||||
mergedAt string // the commit the release merged web at; "" for no merge of the release
|
||||
fpSrc uint64 // the source revision at which web's fingerprint last changed
|
||||
yes, asked bool
|
||||
}{
|
||||
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
|
||||
{"a setting changed too", false, fpOld, commit, 7, false, true},
|
||||
{"a setting changed at the build it runs", true, fpTest, commit, 7, false, true},
|
||||
{"no approved release carries it", true, fpOld, "", 7, false, true},
|
||||
{"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true},
|
||||
// Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing.
|
||||
{"a fingerprint another release moved", true, fpOld, commit, 8, false, true},
|
||||
// Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the
|
||||
// indexer writes fp-src 0, not known, though Src is still the release's: asked of a person.
|
||||
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
|
||||
} {
|
||||
asksKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
|
||||
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
|
||||
if c.mergedAt != "" {
|
||||
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/web", merge)
|
||||
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/other", other)
|
||||
}
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
|
||||
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
|
||||
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
|
||||
p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out)
|
||||
w := out.Wants["g14"]
|
||||
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
|
||||
t.Errorf("%s: the want is %+v", c.name, w)
|
||||
}
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
if _, there := asksKV.values[akey]; there != c.asked {
|
||||
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
|
||||
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
|
||||
if got := p.w.assigned(); len(got) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v", got)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "c", "a")
|
||||
placeOn(t, p.w, "n2", "d")
|
||||
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
|
||||
t.Errorf("the placement alone: assigned %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
|
||||
// kept there while the want stays at its build.
|
||||
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
sync := func() {
|
||||
for k, v := range wantsKV.values {
|
||||
put(delivery.AssignerWantsBucket, k, v)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
p.wants(context.Background())
|
||||
p.nodeWants(context.Background())
|
||||
sync()
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
|
||||
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
|
||||
}
|
||||
nkey, _ := delivery.WantNodeKey("g14")
|
||||
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
|
||||
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
|
||||
}
|
||||
|
||||
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
|
||||
put(delivery.HaveBucket, "g14.web", waits)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
|
||||
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
|
||||
// 8 s, and a want put unchanged each pass made the node compose again each time.
|
||||
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
|
||||
asked := asksKV.values[akey]
|
||||
p.wants(context.Background())
|
||||
if !bytes.Equal(asksKV.values[akey], asked) {
|
||||
t.Fatalf("the record ask the node's have waits on was taken away or made again: %q", asksKV.values[akey])
|
||||
}
|
||||
wantPuts, askPuts := wantsKV.puts, asksKV.puts
|
||||
p.wants(context.Background())
|
||||
if wantsKV.puts != wantPuts || asksKV.puts != askPuts {
|
||||
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
|
||||
}
|
||||
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
|
||||
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
|
||||
if wantsKV.puts != wantPuts {
|
||||
t.Fatalf("a want put unchanged reached the bucket")
|
||||
}
|
||||
a, err := decodeAsk(akey, asksKV.values[akey])
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
sync()
|
||||
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
|
||||
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
}
|
||||
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
|
||||
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", ran)
|
||||
p.wants(context.Background())
|
||||
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
|
||||
t.Errorf("a pass after the move dropped the yes: %+v", got)
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
|
||||
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
|
||||
// cleared once, and said again at a later pass while the findings cannot be read.
|
||||
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
|
||||
var said []string
|
||||
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
|
||||
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
|
||||
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
|
||||
var conds []oc.Condition
|
||||
p.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := 1
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
|
||||
}
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
if len(conds) != 2 || conds[0].State != occ.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0614 §2, §3: of what a module invokes, the pass keeps as edges the verbs the graph does not leave out and another
|
||||
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
|
||||
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
|
||||
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
fp := func(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil {
|
||||
t.Error("with no graph read, a want waits on a verb")
|
||||
}
|
||||
for module, n := range map[string]graph.Node{
|
||||
"caller": {FP: fp("c"), Repository: "novox/caller", Invokes: []string{"seat:x.v", "loop.l", "none.t", "caller.own"}, LeftOut: []string{"loop.l"},
|
||||
Serves: []string{"caller.own"}},
|
||||
"server": {FP: fp("e"), Repository: "novox/server", Serves: []string{"seat:x.v", "seat:x.w"}},
|
||||
"loop": {FP: fp("1"), Repository: "novox/loop", Serves: []string{"loop.l"}},
|
||||
"late": {FP: fp("2"), Repository: "novox/late", Serves: []string{"late.t"}},
|
||||
} {
|
||||
raw, err := graph.Encode(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(graph.Bucket, module, raw)
|
||||
}
|
||||
g := p.moduleGraph()
|
||||
if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil {
|
||||
t.Errorf("the kept verb edges of caller = %v", got)
|
||||
}
|
||||
digest := strings.Repeat("d", 64)
|
||||
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "server", JSON: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`})
|
||||
get := func(_ context.Context, key string) ([]byte, error) {
|
||||
if key == digest {
|
||||
return man, nil
|
||||
}
|
||||
return nil, errNotThere
|
||||
}
|
||||
build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
||||
key, _ := delivery.BuilderBuildKey("server", fp("f"))
|
||||
p.w.put(delivery.BuilderBuildsBucket, key, build)
|
||||
running := delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
at := func(f string) delivery.Have { h := running; h.FP = f; return h }
|
||||
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]delivery.Have{
|
||||
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
|
||||
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
|
||||
if served(v) != want {
|
||||
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an
|
||||
// offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer.
|
||||
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
|
||||
for _, online := range []bool{false, true} {
|
||||
offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online},
|
||||
Haves: map[string]map[string]delivery.Have{"n2": {"bus": at(fp("2"))}}})
|
||||
if offering(graph.BusState) != online {
|
||||
t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// decodedAsk is an asks-state value with the router's ask it carries decoded (mesh-sdk keeps it as raw JSON).
|
||||
type decodedAsk struct {
|
||||
Asking int
|
||||
Ask oc.Ask
|
||||
}
|
||||
|
||||
func decodeAsk(key string, data []byte) (decodedAsk, error) {
|
||||
a, ask, err := assign.DecodeAsk(key, data)
|
||||
return decodedAsk{Asking: a.Asking, Ask: ask}, err
|
||||
}
|
||||
|
||||
func mustAskID(a delivery.AssignerAsk) string {
|
||||
ask, err := assign.AskOf(a)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return ask.ID
|
||||
}
|
||||
|
||||
@@ -10,10 +10,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -47,8 +46,6 @@ type putActs struct {
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
@@ -56,7 +53,7 @@ type putActs struct {
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
@@ -97,9 +94,9 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned := p.w.assigned()
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
@@ -186,24 +183,20 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
if c, ok := chosen[module]; ok {
|
||||
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
|
||||
} else if policy == "" {
|
||||
if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, ask, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
expires := assign.ExpiresAt(ask)
|
||||
w.AskExpired = !expires.IsZero() && w.Now.After(expires)
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -220,16 +213,17 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
@@ -243,13 +237,13 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, _, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -258,15 +252,15 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
|
||||
func (p *passer) performs(key string, act occ.Act) error {
|
||||
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range ask.Options {
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return occ.Performs(o, act)
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
@@ -388,9 +382,8 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and
|
||||
// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274).
|
||||
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
@@ -400,10 +393,10 @@ func (a *putActs) warrant(id, ask string) {
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
|
||||
@@ -1,753 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
|
||||
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
|
||||
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
|
||||
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
|
||||
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
|
||||
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
|
||||
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
|
||||
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
|
||||
|
||||
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
|
||||
type publisher func(event string, body any, id string) error
|
||||
|
||||
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
|
||||
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", delivery.AssignerModule)
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
|
||||
// the controller: placement is the one source (ADR 0567 §8, issue 774).
|
||||
type seatActs struct {
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
|
||||
}
|
||||
|
||||
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
|
||||
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
}
|
||||
ask, _, _ := strings.Cut(warrant, "@")
|
||||
go func() {
|
||||
if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
} else {
|
||||
a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
|
||||
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
|
||||
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
|
||||
func (p *passer) clearStaleCalls() {
|
||||
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
|
||||
return
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.callsCleared = true
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "module.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
|
||||
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
|
||||
if node, module, two := strings.Cut(pair, "."); two {
|
||||
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
|
||||
p.callsCleared = false
|
||||
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
func (p *passer) sayWaits(unbound []string) {
|
||||
c := assign.SeatWaitsCondition(unbound)
|
||||
state := string(c.State) + c.Summary
|
||||
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
|
||||
return
|
||||
}
|
||||
if c.State == occ.StateCleared && p.saidWaits == "" {
|
||||
return // nothing was raised by this process
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key], p.saidWaits = true, state
|
||||
}
|
||||
|
||||
// seatPlacements is every node's placement and the revision of its node key (its legacy key's for a node not yet
|
||||
// converted), as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
|
||||
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
|
||||
return p.w.placement.AllRevisions()
|
||||
}
|
||||
|
||||
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
|
||||
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
|
||||
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
|
||||
var newest time.Time
|
||||
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
|
||||
out[node], revs[node] = c, meta.revision
|
||||
if meta.at.After(newest) {
|
||||
newest = meta.at
|
||||
}
|
||||
}
|
||||
return out, revs, newest
|
||||
}
|
||||
|
||||
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
|
||||
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
|
||||
out := map[string]*delivery.CandidateAnswer{}
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := h.Candidates[id]; ok {
|
||||
out[node] = &a
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placementSeat takes every open proposal one step further.
|
||||
func (p *passer) placementSeat(ctx context.Context) {
|
||||
for _, b := range seatBuckets {
|
||||
if p.kvs[b] == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.saidProposed == nil {
|
||||
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
current, revs := p.seatPlacements()
|
||||
ids := make([]string, 0)
|
||||
raws := p.w.read(delivery.PlacementProposalsBucket)
|
||||
for id := range raws {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
prop, err := delivery.DecodePlacementProposal(raws[id])
|
||||
if err != nil || prop.ID != id {
|
||||
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
|
||||
continue
|
||||
}
|
||||
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
|
||||
}
|
||||
}
|
||||
|
||||
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
|
||||
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
|
||||
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
|
||||
kv := p.kvs[delivery.PlacementPulsesBucket]
|
||||
_ = kv.Delete(ctx, key)
|
||||
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
|
||||
if err != nil {
|
||||
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
|
||||
}
|
||||
|
||||
// pulsing says whether a bound's key stands.
|
||||
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
|
||||
|
||||
// deletePulses deletes every bound of a proposal.
|
||||
func (p *passer) deletePulses(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementPulsesBucket) {
|
||||
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
|
||||
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementPulsesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
|
||||
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
|
||||
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
placed, err := assign.AskOf(a)
|
||||
if err != nil {
|
||||
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
|
||||
return prop, false
|
||||
}
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
|
||||
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
|
||||
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = delivery.ProposalAsked, placed.ID
|
||||
return prop, true
|
||||
}
|
||||
|
||||
// stepProposal takes one proposal one step further.
|
||||
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
|
||||
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
|
||||
p.saidProposed[prop.ID] = true
|
||||
}
|
||||
if prop.Withdrawn != "" {
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
if prop.Seed || prop.HandBack {
|
||||
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
|
||||
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
|
||||
return
|
||||
}
|
||||
if prop.Warrant != nil {
|
||||
p.addNode(ctx, prop, revs)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
return
|
||||
case !p.pulsing(askPulse) && !again:
|
||||
p.endProposal(ctx, prop, "its ask expired unanswered")
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
}
|
||||
cands, candRevs, written := p.candidatesOf(prop.ID)
|
||||
if len(prop.Must) == 0 {
|
||||
p.mustAnswer(ctx, prop, rev, current, cands)
|
||||
return
|
||||
}
|
||||
for _, node := range prop.Must {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // a candidate just written is not heard yet: its watch wakes the next pass
|
||||
}
|
||||
}
|
||||
if written.After(prop.Written) {
|
||||
prop.Written = written
|
||||
}
|
||||
pulsing := map[string]bool{}
|
||||
for _, node := range prop.Must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
pulsing[node] = p.pulsing(k)
|
||||
}
|
||||
}
|
||||
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
|
||||
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
|
||||
return
|
||||
}
|
||||
prop.Verdict = &v
|
||||
switch v.State {
|
||||
case delivery.VerdictFail:
|
||||
prop.State, prop.Ask = delivery.ProposalFailed, ""
|
||||
case delivery.VerdictWaits:
|
||||
prop.State, prop.Ask = delivery.ProposalAnswering, ""
|
||||
case delivery.VerdictPass:
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
}
|
||||
if again {
|
||||
prop.Words = "judged and asked again (again)"
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.writeProposal(ctx, prop, rev) {
|
||||
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
|
||||
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
|
||||
}
|
||||
}
|
||||
|
||||
// sameVerdict says two verdicts say the same, their time aside.
|
||||
func sameVerdict(a, b delivery.PlacementVerdict) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
|
||||
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
|
||||
for node, base := range prop.Base {
|
||||
if revs[node] != base {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
|
||||
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
|
||||
key := fmt.Sprint(prop.ID, prop.Base)
|
||||
if got, ok := p.intended[key]; ok {
|
||||
return got
|
||||
}
|
||||
got, _ := p.compute(prop, current, cands)
|
||||
p.intended[key] = got
|
||||
return got
|
||||
}
|
||||
|
||||
// compute is what the proposal changes on each node by its placement, and the nodes that answer it: every node with a
|
||||
// placement or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
|
||||
// change in its answer.
|
||||
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string) {
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
if c, ok := cands[node]; ok && !c.Reached {
|
||||
laid[node] = c.Placement
|
||||
}
|
||||
}
|
||||
nodes := map[string]bool{}
|
||||
for n := range current {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range p.w.records() {
|
||||
nodes[n] = true
|
||||
}
|
||||
return judge.Intended(current, laid, slices.Sorted(maps.Keys(nodes)))
|
||||
}
|
||||
|
||||
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
|
||||
// change, and writes them into the proposal.
|
||||
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
|
||||
for _, node := range prop.Nodes() {
|
||||
if _, ok := cands[node]; !ok {
|
||||
return // the verbs write the candidates before the proposal: wait to hear them
|
||||
}
|
||||
}
|
||||
intended, must := p.compute(prop, current, cands)
|
||||
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
|
||||
for _, node := range must {
|
||||
if _, ok := cands[node]; ok {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
|
||||
if pl, ok := current[node]; ok {
|
||||
c.Placement = &pl
|
||||
}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
for _, node := range must {
|
||||
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
|
||||
p.pulse(ctx, k, judge.AnswerBound)
|
||||
}
|
||||
}
|
||||
prop.Must = must
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
|
||||
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
|
||||
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
for _, node := range again.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
|
||||
// nodes with a record.
|
||||
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
|
||||
held := map[string]bool{}
|
||||
for key := range p.w.read(delivery.ModulesBucket) {
|
||||
if name, err := delivery.ParseModulesKey(key); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
records := p.w.records()
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, and placement-changed said.
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
}
|
||||
cands, _, _ := p.candidatesOf(prop.ID)
|
||||
laid := map[string]*delivery.Placement{}
|
||||
for _, node := range prop.Nodes() {
|
||||
laid[node] = cands[node].Placement
|
||||
}
|
||||
writes, err := assign.Writes(prop, laid)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
var changes []delivery.PlacementChange
|
||||
writtenLaid := map[string]*delivery.Placement{}
|
||||
after := maps.Clone(current)
|
||||
afterRevs := maps.Clone(revs)
|
||||
for _, w := range writes {
|
||||
rev2, err := p.writeNode(ctx, w.Node, w.Value)
|
||||
if err != nil {
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
before := current[w.Node]
|
||||
now := delivery.Placement{Node: w.Node}
|
||||
if w.Value != nil {
|
||||
now = *w.Value
|
||||
after[w.Node] = *w.Value
|
||||
} else {
|
||||
delete(after, w.Node)
|
||||
}
|
||||
afterRevs[w.Node] = rev2
|
||||
c := delivery.ChangeOf(before, now)
|
||||
c.Before, c.After = w.Base, rev2
|
||||
changes = append(changes, c)
|
||||
writtenLaid[w.Node] = w.Value
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes)
|
||||
}
|
||||
|
||||
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
|
||||
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
|
||||
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
|
||||
// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
|
||||
func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) {
|
||||
current, _ := p.seatPlacements()
|
||||
if moved(prop, revs) {
|
||||
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
|
||||
return
|
||||
}
|
||||
laid, findings := delivery.LayChanges(current, prop.Changes)
|
||||
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
|
||||
node := prop.Changes[0].Node
|
||||
if prop.Changes[0].Op == delivery.OpRemoveNode {
|
||||
p.removeNode(ctx, prop, current, findings)
|
||||
return
|
||||
}
|
||||
value, ok := laid[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, &value)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(delivery.Placement{Node: node}, value)
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
|
||||
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant.
|
||||
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
|
||||
func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) {
|
||||
node := prop.Changes[0].Node
|
||||
before, ok := current[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, nil)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
|
||||
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
|
||||
at = t
|
||||
}
|
||||
}
|
||||
delivery.SortChanges(changes)
|
||||
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
|
||||
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
|
||||
p.w.put(delivery.AssignerAsksBucket, akey, nil)
|
||||
}
|
||||
}
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
// deleteCandidates deletes every candidate of a proposal.
|
||||
func (p *passer) deleteCandidates(ctx context.Context, id string) {
|
||||
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
|
||||
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
|
||||
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
|
||||
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
|
||||
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
|
||||
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
|
||||
raw, err := delivery.EncodePlacementProposal(prop)
|
||||
if err != nil {
|
||||
p.logf("the proposal %s is not written: %v", prop.ID, err)
|
||||
return false
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementProposalsBucket]
|
||||
var at uint64
|
||||
if rev == 0 {
|
||||
at, err = kv.Create(ctx, prop.ID, raw)
|
||||
} else {
|
||||
at, err = kv.Update(ctx, prop.ID, raw, rev)
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
|
||||
return true
|
||||
}
|
||||
|
||||
// say says one of the seat's events, said in the journal when the bus refuses it.
|
||||
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
|
||||
if p.publish == nil {
|
||||
return
|
||||
}
|
||||
if err := p.publish(event, body, id); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// writeNode writes one node's placement after (nil: none) as its keys (mesh-sdk delivery.PlacementKeys.Writes; research
|
||||
// 072 row 37): the module keys that change, each carrying the value it replaces, then the node key, the commit, then the
|
||||
// module keys no longer placed. It answers the node key's revision after (0: deleted). A write that fails up to the
|
||||
// commit ends it with nothing changed for any reader, the module keys written before it read at their previous values;
|
||||
// a delete after the commit that fails leaves a module key the node key does not name, which places nothing and is
|
||||
// deleted at the node's next write. Each key written is taken into the world at once, so the next write of the pass
|
||||
// reads it.
|
||||
func (p *passer) writeNode(ctx context.Context, node string, after *delivery.Placement) (uint64, error) {
|
||||
writes, err := p.w.placement.Writes(node, after)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
kv := p.kvs[delivery.PlacementBucket]
|
||||
committed, rev := false, uint64(0)
|
||||
for _, w := range writes {
|
||||
got, err := writeKey(ctx, kv, w)
|
||||
if err != nil {
|
||||
if committed {
|
||||
p.logf("placement %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
|
||||
continue
|
||||
}
|
||||
return 0, fmt.Errorf("placement %s at revision %d: %w", w.Key, w.Base, err)
|
||||
}
|
||||
p.w.putEntry(delivery.PlacementBucket, w.Key, w.Value, got, p.clock())
|
||||
if w.Commit {
|
||||
committed, rev = true, got
|
||||
}
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// writeKey makes one write of a placement key by compare-and-set and answers the key's revision after (0: deleted).
|
||||
func writeKey(ctx context.Context, kv jetstream.KeyValue, w delivery.PlacementKeyWrite) (uint64, error) {
|
||||
switch {
|
||||
case w.Value == nil:
|
||||
return 0, kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
|
||||
case w.Base == 0:
|
||||
return kv.Create(ctx, w.Key, w.Value)
|
||||
default:
|
||||
return kv.Update(ctx, w.Key, w.Value, w.Base)
|
||||
}
|
||||
}
|
||||
|
||||
// convertPlacement converts every node the state holds only in the form before, `placement.<node>`, into its node key
|
||||
// and module keys (research 072 row 37), once: each node converted is said, and a node whose conversion fails is said
|
||||
// and tried again at the next pass. The legacy key is left for the readers not yet delivered; a node key wins over it
|
||||
// for every reader that reads both, and it is deleted with the node key when the node's placement is.
|
||||
func (p *passer) convertPlacement(ctx context.Context) {
|
||||
if p.convertedAll {
|
||||
return
|
||||
}
|
||||
migrations, err := p.w.placement.Migrations()
|
||||
if err != nil {
|
||||
p.logf("placement: some nodes in the form before cannot be converted, and stay read from it: %v", err)
|
||||
}
|
||||
failed := false
|
||||
for _, node := range slices.Sorted(maps.Keys(migrations)) {
|
||||
before, rev, _ := p.w.placement.Of(node)
|
||||
got, err := p.writeNode(ctx, node, &before)
|
||||
if err != nil {
|
||||
failed = true
|
||||
p.logf("placement: %s is not converted from placement.%s (revision %d) to its node and module keys, and is tried again at the next pass: %v", node, node, rev, err)
|
||||
continue
|
||||
}
|
||||
p.logf("placement: %s converted from placement.%s (revision %d) to %s.node (revision %d) and %d module keys (research 072 row 37); placement.%s stays for readers not yet delivered",
|
||||
node, node, rev, node, got, len(before.Modules), node)
|
||||
}
|
||||
p.convertedAll = !failed && err == nil
|
||||
}
|
||||
@@ -1,620 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
|
||||
|
||||
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
|
||||
type seatStore interface {
|
||||
store
|
||||
del(state, key string) error
|
||||
history(state, key string) ([]stdio.StateRevision, error)
|
||||
}
|
||||
|
||||
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
held := delivery.NewPlacementKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
out, revs := held.AllRevisions()
|
||||
if len(bad) > 0 && len(out) == 0 {
|
||||
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
raw = []byte(c)
|
||||
case nil:
|
||||
return nil, errors.New("propose names its changes")
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// doPropose is the verb propose (§2).
|
||||
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
changes, err := changesOf(args["changes"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warrant, err := warrantOf(args["warrant"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
|
||||
if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
|
||||
func warrantOf(v any) (*delivery.AddNodeWarrant, error) {
|
||||
var raw []byte
|
||||
switch w := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
if strings.TrimSpace(w) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw = []byte(w)
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out delivery.AddNodeWarrant
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
|
||||
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
|
||||
// as proven, which the refusal names.
|
||||
func moduleOfCaller(by string) string {
|
||||
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
|
||||
return delivery.EnrolmentModule
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
|
||||
}
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
}
|
||||
|
||||
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
|
||||
// when its verdict passes.
|
||||
func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
p, err := s.proposal(arg(args, "id"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
History int `json:"history-per-key"`
|
||||
Changes []historyEntry `json:"changes,omitempty"`
|
||||
Older string `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
if arg(args, "seats") == "true" {
|
||||
return s.seats(arg(args, "node"))
|
||||
}
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
}
|
||||
nodes := out.Nodes
|
||||
if node != "" {
|
||||
nodes = []string{node}
|
||||
} else if module == "" {
|
||||
nodes = nil
|
||||
for n := range all {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
revisions, err := s.nodeHistory(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var before sketch
|
||||
for _, r := range revisions {
|
||||
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
|
||||
after := sketch{}
|
||||
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
|
||||
after = sketchOf(r.Key, r.Value)
|
||||
} else {
|
||||
e.Deleted = true
|
||||
}
|
||||
e.Change = after.changeFrom(n, before)
|
||||
e.Change.Before, e.Change.After = 0, r.Revision
|
||||
before = after
|
||||
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := []openProposal{}
|
||||
for _, id := range ids {
|
||||
if want := arg(args, "id"); want != "" && want != id {
|
||||
continue
|
||||
}
|
||||
p, err := s.proposal(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
if arg(args, "id") != "" && len(out) == 0 {
|
||||
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
|
||||
}
|
||||
return map[string]any{"proposals": out}, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
|
||||
// only from a tool of that name), with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
keys, err := s.st.keys(delivery.SeatHoldersState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []delivery.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var sh delivery.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
|
||||
continue
|
||||
}
|
||||
out = append(out, sh)
|
||||
}
|
||||
return map[string]any{"seats": out, "holds": len(keys)}, nil
|
||||
}
|
||||
|
||||
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
|
||||
type keyedRevision struct {
|
||||
stdio.StateRevision
|
||||
Key string
|
||||
}
|
||||
|
||||
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
|
||||
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
|
||||
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
|
||||
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
|
||||
legacy, err := delivery.LegacyPlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodeKey, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []keyedRevision
|
||||
for _, key := range []string{legacy, nodeKey} {
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, r := range revisions {
|
||||
out = append(out, keyedRevision{StateRevision: r, Key: key})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
|
||||
// and holdings, which change every module (delivery.ChangeOf).
|
||||
type sketch struct {
|
||||
modules []delivery.PlacementRef
|
||||
nodeWide string
|
||||
}
|
||||
|
||||
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
|
||||
func sketchOf(key string, value []byte) sketch {
|
||||
_, _, kind, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
|
||||
raw, _ := json.Marshal([]any{pins, holds})
|
||||
return string(raw)
|
||||
}
|
||||
if kind == delivery.PlacementNodeKind {
|
||||
n, err := delivery.DecodePlacementNode(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
|
||||
for _, m := range p.Modules {
|
||||
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
|
||||
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
|
||||
c := delivery.PlacementChange{Node: node}
|
||||
was := map[string]string{}
|
||||
for _, r := range before.modules {
|
||||
was[r.Module] = r.Digest
|
||||
}
|
||||
now := map[string]bool{}
|
||||
nodeWide := before.nodeWide != s.nodeWide && len(before.modules) > 0
|
||||
for _, r := range s.modules {
|
||||
now[r.Module] = true
|
||||
d, ok := was[r.Module]
|
||||
switch {
|
||||
case !ok:
|
||||
c.Added = append(c.Added, r.Module)
|
||||
case nodeWide || d != r.Digest:
|
||||
c.Changed = append(c.Changed, r.Module)
|
||||
}
|
||||
}
|
||||
for _, r := range before.modules {
|
||||
if !now[r.Module] {
|
||||
c.Removed = append(c.Removed, r.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(c.Added)
|
||||
sort.Strings(c.Removed)
|
||||
sort.Strings(c.Changed)
|
||||
return c
|
||||
}
|
||||
@@ -1,573 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
func (s memStore) del(state, key string) error {
|
||||
delete(s[state].values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
v, ok := s[state].values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
st := memStore{}
|
||||
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
for _, m := range []string{"web", "shell"} {
|
||||
st[modulesState].values[m] = []byte("{}")
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
|
||||
t.Errorf("marked = %+v, %v", p, err)
|
||||
}
|
||||
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
|
||||
t.Error("an unknown proposal was withdrawn")
|
||||
}
|
||||
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
t.Error("an unknown change was taken")
|
||||
}
|
||||
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
|
||||
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
move := got.(proposed).Proposal
|
||||
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
|
||||
t.Fatalf("propose = %+v", move)
|
||||
}
|
||||
|
||||
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
|
||||
reads := map[string]bool{}
|
||||
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
|
||||
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
|
||||
t.Errorf("placement of web = %+v, %v", pa, err)
|
||||
} else {
|
||||
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
|
||||
}
|
||||
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
|
||||
t.Error("a node with no key was answered")
|
||||
}
|
||||
ps, err := s.doProposals(map[string]any{"id": move.ID})
|
||||
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
|
||||
t.Errorf("proposals = %+v, %v", ps, err)
|
||||
} else {
|
||||
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
|
||||
}
|
||||
raw0, err := readManifest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range raw0.State {
|
||||
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
|
||||
t.Errorf("no verb reads the seat's state %s", state.Name)
|
||||
}
|
||||
}
|
||||
for _, stream := range raw0.Streams {
|
||||
// The views family is no seat's stream: the readers of the assigner's states follow it (ADR 0615 §5).
|
||||
if stream.Name != viewsStream && !reads["stream "+stream.Name] {
|
||||
t.Errorf("no verb reads the seat's stream %s", stream.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold: %v", err)
|
||||
}
|
||||
// 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with
|
||||
// the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused.
|
||||
addNode := func(by, warrant string) error {
|
||||
_, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace",
|
||||
"decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by)
|
||||
return err
|
||||
}
|
||||
good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`
|
||||
for _, c := range []struct{ by, warrant, want string }{
|
||||
{"novox.mesh-grants", good, "mesh-enrolment's alone"},
|
||||
{"person.mesh-enrolment", good, "mesh-enrolment's alone"},
|
||||
{"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
{"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"},
|
||||
{"novox.mesh-enrolment", "", "carries no warrant"},
|
||||
} {
|
||||
if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want)
|
||||
}
|
||||
}
|
||||
if err := addNode("novox.mesh-enrolment", good); err != nil {
|
||||
t.Errorf("an add-node from mesh-enrolment: %v", err)
|
||||
}
|
||||
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
|
||||
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
|
||||
// of a node running modules, or with another node's warrant, it is refused.
|
||||
joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}})
|
||||
st[delivery.PlacementState].values["placement.joiner"] = joiner
|
||||
removeNode := func(node, warrant string) (any, error) {
|
||||
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
|
||||
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
|
||||
}
|
||||
for _, c := range []struct{ node, warrant, want string }{
|
||||
{"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"},
|
||||
{"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
} {
|
||||
if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want)
|
||||
}
|
||||
}
|
||||
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
|
||||
t.Errorf("a remove-node from mesh-enrolment: %v", err)
|
||||
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
|
||||
t.Errorf("the remove-node proposed %+v", rm)
|
||||
}
|
||||
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
|
||||
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey, _ := delivery.LegacyPlacementKey("ace")
|
||||
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
|
||||
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
|
||||
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
hk, _ := delivery.HaveNodeKey(node)
|
||||
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
|
||||
}
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Errorf("ace: %+v %v", d, ok)
|
||||
}
|
||||
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
|
||||
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
}
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
p.unplacedFindings()
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey2, _ := delivery.LegacyPlacementKey("g14")
|
||||
w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now())
|
||||
p.unplacedFindings()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("placed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
|
||||
writes := 0
|
||||
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
|
||||
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}}
|
||||
place := func(node string, holds ...delivery.Holding) {
|
||||
pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := delivery.LegacyPlacementKey(node)
|
||||
p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now())
|
||||
}
|
||||
place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
ctx := context.Background()
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
|
||||
t.Fatalf("%d writes: %v", writes, kv.values)
|
||||
}
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 {
|
||||
t.Errorf("an unchanged placement wrote %d keys", writes-3)
|
||||
}
|
||||
place("g14") // g14 holds nothing now
|
||||
p.seatHolders(ctx)
|
||||
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
|
||||
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
|
||||
}
|
||||
st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}}
|
||||
out, err := newSeatTools(st, nil).seats("ace")
|
||||
if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 {
|
||||
t.Errorf("seats on ace = %+v, %v", out, err)
|
||||
}
|
||||
})
|
||||
t.Run("research 072 row 37: the form before is converted once, a node is written as its keys, and history reads both", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var logs []string
|
||||
p := &passer{w: newWorld(), kvs: buckets{delivery.PlacementBucket: kv}, now: time.Now,
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
put := func(key string, value []byte, rev uint64) {
|
||||
p.w.putEntry(delivery.PlacementBucket, key, value, rev, time.Now())
|
||||
}
|
||||
was := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
raw, _ := delivery.EncodePlacement(was)
|
||||
put("placement.anchor", raw, kv.write("placement.anchor", raw))
|
||||
p.convertPlacement(ctx)
|
||||
if strings.Join(kv.log, " ") != "create anchor.web create anchor.node" || !p.convertedAll ||
|
||||
!strings.Contains(strings.Join(logs, "\n"), "anchor converted from placement.anchor") {
|
||||
t.Fatalf("the conversion wrote %v, said %v", kv.log, logs)
|
||||
}
|
||||
p.convertPlacement(ctx)
|
||||
if len(kv.log) != 2 {
|
||||
t.Errorf("a converted node is converted again: %v", kv.log)
|
||||
}
|
||||
// A write: the module keys, the node key (its commit), the mirror, then the module key no longer placed.
|
||||
after := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2222}}}}
|
||||
kv.log = nil
|
||||
rev, err := p.writeNode(ctx, "anchor", &after)
|
||||
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node update placement.anchor delete anchor.web" || rev != kv.revs["anchor.node"] {
|
||||
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
|
||||
}
|
||||
if got, _ := p.seatPlacements(); !delivery.SamePlacement(got["anchor"], after) {
|
||||
t.Errorf("the write is not read back at once: %+v", got)
|
||||
}
|
||||
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
|
||||
kv.refuse, kv.log = "anchor.node", nil
|
||||
moved := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2223}}}}
|
||||
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
|
||||
t.Fatalf("a refused commit = %v, %v", kv.log, err)
|
||||
}
|
||||
fresh, err := delivery.PlacementsOf(kv.values)
|
||||
if err != nil || !delivery.SamePlacement(fresh["anchor"], after) {
|
||||
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
|
||||
}
|
||||
kv.refuse = ""
|
||||
// The verb's history reads the form before, then the node key.
|
||||
st := memStore{delivery.PlacementState: &memKV{values: kv.values}}
|
||||
hs := newSeatTools(st, nil)
|
||||
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
|
||||
"placement.anchor": {{Key: "placement.anchor", Value: raw, Revision: 1, Op: "put"}},
|
||||
"anchor.node": {{Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
|
||||
}}
|
||||
pa, err := hs.doPlacement(map[string]any{"node": "anchor", "history": "true"})
|
||||
a, _ := pa.(placementAnswer)
|
||||
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
|
||||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
|
||||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
|
||||
t.Errorf("history = %+v, %v", pa, err)
|
||||
}
|
||||
// No placement: the mirror first, then the node key, then the module keys.
|
||||
kv.log = nil
|
||||
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete placement.anchor delete anchor.node delete anchor.shell" {
|
||||
t.Errorf("the delete = %v, %v", kv.log, err)
|
||||
}
|
||||
})
|
||||
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil }}}
|
||||
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now())
|
||||
p.kvs[provisions.ProvisionsBucket] = &memKV{}
|
||||
of := p.provisionsOf()
|
||||
if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed {
|
||||
t.Fatalf("provisions = %+v", got)
|
||||
}
|
||||
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
|
||||
t.Fatalf("findings = %v", findings)
|
||||
}
|
||||
provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
|
||||
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided})
|
||||
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now())
|
||||
p.provisionsOf()
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings after provided = %v", findings)
|
||||
}
|
||||
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
|
||||
ctx := context.Background()
|
||||
p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
|
||||
gone := map[string]bool{"ace.photos.s3": true}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
|
||||
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if len(findings) != 0 {
|
||||
t.Fatal("said before its bound")
|
||||
}
|
||||
p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
|
||||
p.provisionsMissing(ctx, gone)
|
||||
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
|
||||
t.Fatalf("past its bound: findings %v", findings)
|
||||
}
|
||||
p.provisionsMissing(ctx, map[string]bool{})
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("keyed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
|
||||
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
|
||||
func wantsVerbs(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||
st := memStore{}
|
||||
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
|
||||
// a fingerprint, and laptop's web refused there.
|
||||
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
|
||||
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
|
||||
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
|
||||
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
|
||||
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
|
||||
manifest := func(name, extra string) {
|
||||
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
|
||||
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
|
||||
}
|
||||
manifest("web", "")
|
||||
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
|
||||
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
var asked []string
|
||||
var said []string
|
||||
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
}, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
|
||||
policyOf := func(module string) policyRow {
|
||||
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(map[string]any)["modules"].([]policyRow)[0]
|
||||
}
|
||||
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
|
||||
t.Errorf("photos = %+v", p)
|
||||
}
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { // the policy its manifest says (issue 774)
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
|
||||
// controller being gone (1126/B8).
|
||||
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
|
||||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
|
||||
t.Errorf("compare = %+v", got)
|
||||
}
|
||||
pub, _ := out.(map[string]any)["published"].(map[string][]publishedRow)
|
||||
if a, l := pub["anchor"], pub["laptop"]; len(a) != 2 || a[0].Module != "node" || a[1].Module != "web" || a[1].FP != fpNew ||
|
||||
!strings.HasPrefix(a[1].Digest, "sha256:") || len(l) != 1 || l[0].Refused != "a required value is unset" || l[0].Digest != "" {
|
||||
t.Errorf("published = %+v (unread %v)", pub, out.(map[string]any)["published-unread"])
|
||||
}
|
||||
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
|
||||
// large states (the modules, their manifests, the composed record).
|
||||
watches := map[string]int{}
|
||||
byWatch := newWantsTools(st, nil, nil)
|
||||
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
|
||||
watches[state]++
|
||||
for k, v := range st[state].values {
|
||||
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true})
|
||||
}
|
||||
return nil
|
||||
}, func(state, key string) ([]byte, bool, error) {
|
||||
watches[state+" by key"]++
|
||||
v, ok := st[state].values[key]
|
||||
return v, ok, nil
|
||||
})
|
||||
for range 2 {
|
||||
wout, werr := byWatch.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("compare by watch = %+v", got)
|
||||
}
|
||||
}
|
||||
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
|
||||
t.Errorf("the manifests were read by key %d times, want each once", n)
|
||||
}
|
||||
for _, state := range []string{modulesState, haveState, health.FindingsState} {
|
||||
if watches[state] != 1 {
|
||||
t.Errorf("%s was watched %d times, want once", state, watches[state])
|
||||
}
|
||||
}
|
||||
// ADR 0580 clause 3: the compare reads through mesh-sdk's watched reader and the seat keeps no mirror of its own.
|
||||
for f := range reflect.TypeFor[wantsTools]().Fields() {
|
||||
if f.Type.Kind() == reflect.Map || (f.Type.Kind() == reflect.Pointer && f.Type.Elem().Kind() == reflect.Map) {
|
||||
t.Errorf("wantsTools keeps %s, a copy of its own beside the watched reader", f.Name)
|
||||
}
|
||||
}
|
||||
if watches[policiesState] != 0 {
|
||||
t.Errorf("the seat's own small policies state was watched: %v", watches)
|
||||
}
|
||||
// a person's record for web: written, recorded, said.
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chosen, _ := assign.DecodePolicies(st[policiesState].values)
|
||||
if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") ||
|
||||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
|
||||
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
|
||||
}
|
||||
// A person may choose the bus's policy as any module's (ADR 0601 §1: no part treats the module that provides the bus apart).
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err != nil {
|
||||
t.Errorf("a person's policy for the bus was refused: %v", err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "default", "why": "back to the module"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
|
||||
t.Error("a policy without why was set")
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 {
|
||||
t.Errorf("default left %v (%v)", st[policiesState].values, err)
|
||||
}
|
||||
// bus: the node behind the indexer's newest.
|
||||
b, err := w.doBus(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodes := b.(map[string]any)["nodes"].([]busNode)
|
||||
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
|
||||
t.Errorf("bus = %+v, asked %v", b, asked)
|
||||
}
|
||||
if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
}
|
||||
|
||||
// readManifest is the module's manifest, read as the controller reads it.
|
||||
func readManifest() (catalogue.Manifest, error) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.Decode(raw)
|
||||
}
|
||||
|
||||
// countingKV is a memKV that counts its puts.
|
||||
type countingKV struct {
|
||||
memKV
|
||||
writes *int
|
||||
}
|
||||
|
||||
func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
*c.writes++
|
||||
return c.memKV.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// revKV is a bucket with real revisions, refusing a write at a revision that is not the key's, and a write of refuse;
|
||||
// log is every write in order.
|
||||
type revKV struct {
|
||||
jetstream.KeyValue
|
||||
rev uint64
|
||||
values map[string][]byte
|
||||
revs map[string]uint64
|
||||
log []string
|
||||
refuse string
|
||||
}
|
||||
|
||||
func (k *revKV) write(key string, value []byte) uint64 {
|
||||
k.rev++
|
||||
k.values[key], k.revs[key] = value, k.rev
|
||||
return k.rev
|
||||
}
|
||||
|
||||
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if _, ok := k.values[key]; ok || key == k.refuse {
|
||||
return 0, errors.New("key exists")
|
||||
}
|
||||
k.log = append(k.log, "create "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if k.revs[key] != revision || key == k.refuse {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "update "+key)
|
||||
return k.write(key, value), nil
|
||||
}
|
||||
|
||||
func (k *revKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
if key == k.refuse {
|
||||
return errors.New("wrong last sequence")
|
||||
}
|
||||
k.log = append(k.log, "delete "+key)
|
||||
delete(k.values, key)
|
||||
delete(k.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// historyStore is a memStore whose history is given.
|
||||
type historyStore struct {
|
||||
memStore
|
||||
revisions map[string][]stdio.StateRevision
|
||||
}
|
||||
|
||||
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
|
||||
return s.revisions[key], nil
|
||||
}
|
||||
@@ -14,17 +14,12 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
@@ -356,17 +351,7 @@ func serveTools() error {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
|
||||
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
|
||||
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
|
||||
return stdio.SeatPublish(wantsEventSubject(e), b, id)
|
||||
})
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
tools = append(tools, wants.tools()...)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
|
||||
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
|
||||
return stdio.Serve("", tools)
|
||||
return stdio.Serve("", s.tools())
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
|
||||
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
|
||||
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
|
||||
// manifests, as a provider is counted (ADR 0603 §5).
|
||||
|
||||
// moduleGraph is the module graph as the indexer's first view and views family last said it (ADR 0631); a key that
|
||||
// does not decode is left out, so its module waits on no verb.
|
||||
func (p *passer) moduleGraph() graph.Graph {
|
||||
g := graph.Graph{}
|
||||
for key, raw := range p.w.read(graph.Bucket) {
|
||||
module, err := graph.ParseKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(raw); err == nil {
|
||||
g[module] = n
|
||||
}
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
|
||||
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
|
||||
// no node of module.
|
||||
func keptInvokes(g graph.Graph, module string) []string {
|
||||
n, ok := g[module]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, v := range n.Invokes {
|
||||
if slices.Contains(n.LeftOut, v) {
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
|
||||
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
|
||||
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
|
||||
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
|
||||
// a verb.
|
||||
func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
if len(g) == 0 {
|
||||
return nil
|
||||
}
|
||||
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
|
||||
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A
|
||||
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
|
||||
// holds a consumer for good.
|
||||
func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
|
||||
offered := map[string]bool{}
|
||||
for node, haves := range n.Haves {
|
||||
if !n.Online[node] {
|
||||
continue
|
||||
}
|
||||
for module, h := range haves {
|
||||
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
|
||||
continue
|
||||
}
|
||||
m, ok := p.manifestOf(ctx, get, module, h.FP)
|
||||
if !ok {
|
||||
m = trunk[module]
|
||||
}
|
||||
for _, o := range m.Offers() {
|
||||
offered[o] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return offered[provision] }
|
||||
}
|
||||
|
||||
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
|
||||
func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) {
|
||||
m, ok := p.manifestOf(ctx, get, module, fp)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return graph.ServesOf(m), true
|
||||
}
|
||||
|
||||
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
|
||||
// and whether it could be read.
|
||||
func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
raw := p.w.value(delivery.BuilderBuildsBucket, key)
|
||||
if raw == nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.Manifest == "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
got := p.w.manifest(ctx, get, b.Manifest)
|
||||
if got.err != "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
return got.manifest, true
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/heard"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// viewsStream is the assigner's views family on its old streams section, beside placement (ADR 0619 §2, ADR 0615 §5),
|
||||
// as its manifest's `streams` names it; indexerViewsStream is the indexer's, as the indexer's manifest names it.
|
||||
const (
|
||||
viewsStream = "assigner-views"
|
||||
indexerViewsStream = "indexer-views"
|
||||
)
|
||||
|
||||
// marksBucket is the assigner's state of what its views family said each state up to.
|
||||
var marksBucket = delivery.AssignerModule + "_" + views.MarksState
|
||||
|
||||
// viewedTool is one tool over one state of the assigner's.
|
||||
type viewedTool struct{ tool, state, words string }
|
||||
|
||||
// firstViewTools are the assigner's first-view tools (ADR 0621 §1): each answers only its callers list
|
||||
// (views.CallersEnv), which the manifest writes.
|
||||
var firstViewTools = []viewedTool{
|
||||
{"placement", delivery.PlacementState, "where each module is placed: the seat mesh-placement's placement, by module."},
|
||||
{"seat-holders", delivery.SeatHoldersState, "who holds which seat on which node."},
|
||||
{"wants", "wants", "what each node should run: want.<node>.<module>, its fingerprint and the rule that gave it."},
|
||||
}
|
||||
|
||||
// readTools are the assigner's ordinary read tools, one per state its readers read (design 25 § 12.2).
|
||||
var readTools = []viewedTool{
|
||||
{"assigner_read_placement", delivery.PlacementState, "where each module is placed, by module."},
|
||||
{"assigner_read_seat_holders", delivery.SeatHoldersState, "who holds which seat on which node."},
|
||||
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
|
||||
{"assigner_read_findings", health.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
|
||||
{"assigner_read_asks", "asks", "the asks the assigner put to a person, and their answers."},
|
||||
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
|
||||
}
|
||||
|
||||
// viewedStates are the states the views family says, each by its bucket: the six its readers read.
|
||||
var viewedStates = map[string]string{
|
||||
delivery.PlacementState: delivery.PlacementBucket,
|
||||
delivery.SeatHoldersState: delivery.SeatHoldersBucket,
|
||||
"wants": delivery.AssignerWantsBucket,
|
||||
health.FindingsState: assign.FindingsBucket,
|
||||
"asks": delivery.AssignerAsksBucket,
|
||||
"differences": delivery.AssignerDifferencesBucket,
|
||||
}
|
||||
|
||||
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
|
||||
// runtime of its own is connected (ADR 0621 §3).
|
||||
func viewTools(getenv func(string) string, state func(string) views.Source, now func() time.Time) []stdio.Tool {
|
||||
marks := state(views.MarksState)
|
||||
var out []stdio.Tool
|
||||
for _, t := range firstViewTools {
|
||||
out = append(out, views.FirstViewTool(delivery.AssignerModule, t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now))
|
||||
}
|
||||
for _, t := range readTools {
|
||||
out = append(out, views.ReadTool(delivery.AssignerModule, t.tool, t.state, t.words, state(t.state), marks, now))
|
||||
}
|
||||
// Its own first views, of the indexer's manifests, modules and graph (DLV-F3r, ADR 0631 §2): read, and asked again by hand.
|
||||
return append(out, views.StatusTools(delivery.AssignerModule, "assigner_first_views", "assigner_first_view_again", stdio.State(views.StatusState), handacts.Recorder{Ask: stdio.Ask}, now)...)
|
||||
}
|
||||
|
||||
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
|
||||
|
||||
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
|
||||
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
|
||||
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs buckets, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
|
||||
tooLarge := func(state, key string, size int) {
|
||||
if findings.Put == nil {
|
||||
return
|
||||
}
|
||||
k, v, err := views.TooLargeFinding(delivery.AssignerModule, state, key, size, time.Now())
|
||||
if err == nil {
|
||||
err = findings.Put(k, v)
|
||||
}
|
||||
if err != nil {
|
||||
logf("views: the finding of %s.%s too large is not written: %v", state, key, err)
|
||||
}
|
||||
}
|
||||
process.AwaitBucket(ctx, js, nc, marksBucket, "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
|
||||
em := viewsEmitter(js, account, mk, findings, tooLarge, logf)
|
||||
for _, state := range []string{"asks", "differences", health.FindingsState, delivery.PlacementState, delivery.SeatHoldersState, "wants"} {
|
||||
state := state
|
||||
start := func(kv jetstream.KeyValue) {
|
||||
if err := em.Start(ctx, state, kv, errs); err != nil {
|
||||
logf("%v: its readers ask its first view at each change until the assigner starts again", err)
|
||||
}
|
||||
}
|
||||
if kv := kvs[viewedStates[state]]; kv != nil {
|
||||
start(kv)
|
||||
continue
|
||||
}
|
||||
process.AwaitBucket(ctx, js, nc, viewedStates[state], "its changes are not said on the views family", logf).Ready(start)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
|
||||
// view-unsaid, in its findings (review round 2); with no findings state it is said in the log only.
|
||||
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, findings assign.Findings, tooLarge func(string, string, int), logf func(string, ...any)) views.Emitter {
|
||||
em := views.Emitter{Owner: delivery.AssignerModule, Subject: account.StreamSubject, Publish: views.JetStreamPublish(js), Marks: mk, TooLarge: tooLarge, Logf: logf}
|
||||
if findings.Put != nil && findings.Delete != nil {
|
||||
em.Findings = assignerFindings{findings}
|
||||
}
|
||||
return em
|
||||
}
|
||||
|
||||
// assignerFindings is the assigner's findings as views.Emitter's Findings.
|
||||
type assignerFindings struct{ f assign.Findings }
|
||||
|
||||
func (a assignerFindings) Put(key string, value []byte) error { return a.f.Put(key, value) }
|
||||
func (a assignerFindings) Delete(key string) error { return a.f.Delete(key) }
|
||||
|
||||
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
|
||||
var firstViewsBucket = delivery.AssignerModule + "_" + views.StatusState
|
||||
|
||||
// indexerViews are the assigner's first views of the indexer's manifests, modules and module graph (ADR 0615 §1,
|
||||
// ADR 0621 §2, ADR 0631; DLV-F2r): asked at the start through the indexer's tools, then followed on indexer-views, asking once more on a gap;
|
||||
// past the bound its finding names the indexer and the tool, cleared by the answer.
|
||||
type indexerViews struct {
|
||||
js jetstream.JetStream
|
||||
account process.Account
|
||||
call func(string, any) (json.RawMessage, error)
|
||||
findings assign.Findings
|
||||
status *process.AwaitedBucket
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// indexerFirstViews are the indexer's states the assigner takes by first view and the views family, each by the name of
|
||||
// the indexer's bucket the world keeps it under: manifests and modules (ADR 0621 §2), and the module graph, whose verb
|
||||
// edges a want waits on (ADR 0614 §2, §3; ADR 0631 §1).
|
||||
var indexerFirstViews = map[string]string{"manifests": delivery.ManifestsBucket, "modules": delivery.ModulesBucket, graph.State: graph.Bucket}
|
||||
|
||||
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
|
||||
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
|
||||
// seconds). A test shortens them.
|
||||
var firstViewBound, firstViewWait time.Duration
|
||||
|
||||
// start asks the three first views, each answer taken into w under the indexer's bucket names and settled once (a pass waits
|
||||
// for it), and hears the views family; it answers how many settles to wait for.
|
||||
func (iv *indexerViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
|
||||
finding := func(s views.Status, raised bool) {
|
||||
if iv.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
key, value, err := views.UnansweredFinding(delivery.AssignerModule, s, time.Now())
|
||||
if err == nil && raised {
|
||||
err = iv.findings.Put(key, value)
|
||||
} else if err == nil {
|
||||
err = iv.findings.Delete(key)
|
||||
}
|
||||
if err != nil {
|
||||
iv.logf("views: the finding of the first view %s is not written: %v", s.Tool, err)
|
||||
}
|
||||
}
|
||||
followers := map[string]*views.Follower{}
|
||||
var all []*views.Follower
|
||||
for state, bucket := range indexerFirstViews {
|
||||
bucket := bucket
|
||||
tool := delivery.IndexerModule + "." + state
|
||||
f := &views.Follower{Owner: delivery.IndexerModule, Tool: tool, State: state, Ask: views.AskVia(iv.call, tool), Logf: iv.logf,
|
||||
Bound: firstViewBound, Wait: firstViewWait,
|
||||
Raise: func(s views.Status) { finding(s, true) }, Clear: func(s views.Status) { finding(s, false) },
|
||||
Changed: func(key string, value json.RawMessage, deleted bool, at time.Time) {
|
||||
if deleted {
|
||||
value = nil
|
||||
}
|
||||
w.putEntry(bucket, key, value, 0, at)
|
||||
wake()
|
||||
}}
|
||||
followers[state] = f
|
||||
all = append(all, f)
|
||||
go func() {
|
||||
if f.Start(ctx) == nil {
|
||||
settled <- struct{}{}
|
||||
}
|
||||
}()
|
||||
}
|
||||
if iv.status != nil {
|
||||
iv.status.Ready(func(kv jetstream.KeyValue) {
|
||||
for _, f := range all {
|
||||
f.Said = views.KeepStatus(ctx, kv, iv.logf)
|
||||
}
|
||||
if err := views.HearAgain(ctx, kv, all, iv.logf); err != nil {
|
||||
iv.logf("views: assigner_first_view_again is not heard (%v) until the assigner starts again", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// Not heard, the indexer's views family is the finding family-not-heard until it binds (ADR 0615 §5).
|
||||
stream := catalogue.StreamBusName(indexerViewsStream)
|
||||
_ = notHeardWriter(iv.findings, iv.logf).Bind(stream, "the indexer's manifests, modules and graph are the first views' alone until the assigner starts again", func() error {
|
||||
cons, err := iv.account.EventConsumer(ctx, iv.js, stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s on %s does not exist yet: %w", iv.account.EventConsumerName(), stream, err)
|
||||
}
|
||||
_, err = process.HearEvents(cons, views.Hear(ctx, followers, iv.logf), iv.logf)
|
||||
return err
|
||||
})
|
||||
return len(all)
|
||||
}
|
||||
|
||||
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
|
||||
// findings state it is said in the log alone.
|
||||
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
|
||||
w := heard.Writer{Observer: delivery.AssignerModule, Logf: logf, Put: findings.Put}
|
||||
if findings.Delete != nil {
|
||||
w.Delete = func(key string) error {
|
||||
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
|
||||
return nil // none stands
|
||||
}
|
||||
return findings.Delete(key)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
busStepCause = "bus-upgrade"
|
||||
)
|
||||
|
||||
// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit.
|
||||
type wantsTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
emit func(event, id string, body any) error
|
||||
now func() time.Time
|
||||
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
|
||||
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
|
||||
// a large state (issue 1953 L5f). Nil reads key by key.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
// wantsEventSubject is the subject of one of the seat's events.
|
||||
func wantsEventSubject(event string) string {
|
||||
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
|
||||
}
|
||||
|
||||
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Values(state)
|
||||
}
|
||||
return s.direct(state)
|
||||
}
|
||||
|
||||
// direct reads a state key by key, for a small state of the seat's own.
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
|
||||
// under its digest.
|
||||
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Once(manifestsState, digest)
|
||||
}
|
||||
raw, _, found, err := s.st.get(manifestsState, digest)
|
||||
return raw, found, err
|
||||
}
|
||||
|
||||
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
|
||||
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
|
||||
mods, err := s.values(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
|
||||
for name, raw := range mods {
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
held[name] = m
|
||||
if m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mraw, found, err := s.manifest(m.Manifest)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(mraw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
|
||||
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
|
||||
out[name] = man
|
||||
}
|
||||
}
|
||||
return out, held, nil
|
||||
}
|
||||
|
||||
// policyRow is one module's policy as upgrade answers it.
|
||||
type policyRow struct {
|
||||
Module string `json:"module"`
|
||||
Policy string `json:"policy"`
|
||||
From string `json:"from"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
|
||||
// back with default; with compare each node's composition and the assigner's findings.
|
||||
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
manifests, _, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
}
|
||||
out := map[string]any{"composed": composed}
|
||||
// And what every node published (ADR 0593 §8), read only to compare.
|
||||
if values, err := s.values(delivery.ComposedRead); err != nil {
|
||||
out["published-unread"] = err.Error()
|
||||
} else {
|
||||
out["published"] = published(values)
|
||||
}
|
||||
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
} else {
|
||||
keys := []string{}
|
||||
for k := range found {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out["findings"] = keys
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
module, policy := arg(args, "module"), arg(args, "policy")
|
||||
if policy != "" {
|
||||
return s.setPolicy(module, policy, arg(args, "why"), by, manifests)
|
||||
}
|
||||
var rows []policyRow
|
||||
for name, m := range manifests {
|
||||
if module != "" && name != module {
|
||||
continue
|
||||
}
|
||||
p, from, why, who := assign.PolicyOf(m, chosen)
|
||||
rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who})
|
||||
}
|
||||
if module != "" && len(rows) == 0 {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
|
||||
return map[string]any{"modules": rows, "unreadable": unreadable}, nil
|
||||
}
|
||||
|
||||
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
|
||||
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
|
||||
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("a policy is set for one module: name it")
|
||||
}
|
||||
if _, ok := manifests[module]; !ok {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("a person's policy says why")
|
||||
}
|
||||
if by == "" || by == stdio.Unproven {
|
||||
return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller")
|
||||
}
|
||||
what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy)
|
||||
if policy == "default" {
|
||||
if err := s.st.del(policiesState, module); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err)
|
||||
}
|
||||
} else {
|
||||
p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()}
|
||||
if err := p.Check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _ := json.Marshal(p)
|
||||
if err := s.st.put(policiesState, module, raw); err != nil {
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := handacts.Recorder{Ask: s.ask}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
recorded = "not recorded: " + rerr.Error()
|
||||
}
|
||||
body := map[string]any{"module": module, "policy": policy, "why": why, "by": by}
|
||||
said := ""
|
||||
if s.emit != nil {
|
||||
id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10)
|
||||
if err := s.emit(PolicyChangedEvent, id, body); err != nil {
|
||||
said = "not said: " + err.Error()
|
||||
}
|
||||
}
|
||||
return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil
|
||||
}
|
||||
|
||||
// busNode is one node's bus as bus answers it.
|
||||
type busNode struct {
|
||||
Node string `json:"node"`
|
||||
FP string `json:"fp"`
|
||||
Behind bool `json:"behind"`
|
||||
}
|
||||
|
||||
// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind,
|
||||
// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only.
|
||||
func (s *wantsTools) doBus(map[string]any) (any, error) {
|
||||
manifests, held, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus := ""
|
||||
for name, m := range manifests {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
bus = name
|
||||
}
|
||||
}
|
||||
if bus == "" {
|
||||
return nil, errors.New("the indexer holds no module that provides the bus")
|
||||
}
|
||||
newest := held[bus].FP2
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nodes []busNode
|
||||
for key, raw := range haves {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode || module != bus {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHave(raw)
|
||||
if err != nil || h.Phase == delivery.PhaseRemoved {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
|
||||
}
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
|
||||
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
|
||||
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
|
||||
out["steps-unread"] = err.Error()
|
||||
} else {
|
||||
out["steps"] = steps
|
||||
}
|
||||
puts, err := s.values(putsState)
|
||||
if err != nil {
|
||||
out["puts-unread"] = err.Error()
|
||||
} else {
|
||||
mine := map[string]json.RawMessage{}
|
||||
for k, v := range puts {
|
||||
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
|
||||
mine[k] = v
|
||||
}
|
||||
}
|
||||
out["puts"] = mine
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
|
||||
func (s *wantsTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
|
||||
"bus": s.doBus,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,10 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -21,6 +24,18 @@ import (
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -31,22 +46,6 @@ type world struct {
|
||||
healthy map[string][]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
|
||||
trunkBuilt map[string]delivery.BuildLogEntry
|
||||
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
|
||||
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
placement *delivery.PlacementKeys
|
||||
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
|
||||
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
|
||||
sourceGroups map[uint64]string
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
type entryMeta struct {
|
||||
revision uint64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -57,69 +56,7 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(),
|
||||
sourceGroups: map[uint64]string{}}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
|
||||
w.put(bucket, key, value)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if value == nil {
|
||||
delete(w.meta, bucket+"|"+key)
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
}
|
||||
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
|
||||
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
|
||||
w.sourceGroups[revision] = s.Group
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
|
||||
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
|
||||
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
|
||||
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
|
||||
// unknown: none, and the move is asked of a person.
|
||||
func (w *world) release(module, fp string) (string, bool) {
|
||||
t, ok := w.trunk()[module]
|
||||
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
|
||||
return "", false
|
||||
}
|
||||
w.mu.Lock()
|
||||
group := w.sourceGroups[t.FPSrc]
|
||||
w.mu.Unlock()
|
||||
key, err := delivery.MergesKey(group)
|
||||
if group == "" || err != nil {
|
||||
return "", false
|
||||
}
|
||||
raw := w.value(delivery.MergesBucket, key)
|
||||
if raw == nil {
|
||||
return "", false
|
||||
}
|
||||
merge, err := delivery.DecodeMerge(raw)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, pull := range merge.Pulls {
|
||||
if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) {
|
||||
return group, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
func (w *world) metaOf(bucket, key string) entryMeta {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.meta[bucket+"|"+key]
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -129,20 +66,12 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
if w.values[bucket] == nil {
|
||||
w.values[bucket] = map[string][]byte{}
|
||||
}
|
||||
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
|
||||
if value == nil {
|
||||
delete(w.values[bucket], key)
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, nil, 0, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.PlacementBucket {
|
||||
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
|
||||
_, _ = w.placement.Put(key, value, 0, false)
|
||||
}
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -198,19 +127,6 @@ func kvGetter(kv jetstream.KeyValue) getter {
|
||||
}
|
||||
}
|
||||
|
||||
// getter reads a key of a bucket the world holds: errNotThere where it holds none.
|
||||
func (w *world) getter(bucket string) getter {
|
||||
return func(_ context.Context, key string) ([]byte, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
v, ok := w.values[bucket][key]
|
||||
if !ok {
|
||||
return nil, errNotThere
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
|
||||
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
|
||||
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
|
||||
w.mu.Lock()
|
||||
@@ -233,10 +149,10 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
m, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
p = parsed{err: err.Error()}
|
||||
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
|
||||
// module's value where it is read (world.mesh).
|
||||
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
|
||||
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
|
||||
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
|
||||
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
|
||||
p = parsed{module: m.Module, err: err.Error()}
|
||||
} else {
|
||||
p = parsed{module: m.Module, manifest: man}
|
||||
}
|
||||
@@ -246,42 +162,21 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
for node, p := range w.placements() {
|
||||
out[node] = p.Assigned()
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
}
|
||||
return out, len(w.read(delivery.PlacementBucket)) > 0
|
||||
}
|
||||
|
||||
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
|
||||
func (w *world) placements() map[string]delivery.Placement {
|
||||
out, _ := w.placement.AllRevisions()
|
||||
return out
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string]bool, len(placed))
|
||||
for node := range placed {
|
||||
out[node] = true
|
||||
}
|
||||
return out
|
||||
return out, all
|
||||
}
|
||||
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
@@ -299,35 +194,75 @@ func (w *world) trunk() map[string]delivery.Module {
|
||||
return out
|
||||
}
|
||||
|
||||
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
|
||||
// with those placement names (ADR 0593 §8).
|
||||
func (w *world) records() map[string]delivery.NodeRecord {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned, _ = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
m.Digests[name] = mod.Manifest
|
||||
p := w.manifest(ctx, manifests, mod.Manifest)
|
||||
if p.err != "" {
|
||||
m.Unreadable[name] = p.err
|
||||
continue
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
return out
|
||||
return m
|
||||
}
|
||||
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
|
||||
definedIn := map[string]map[string]bool{}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository == assign.SDKRepository {
|
||||
ch.SDKHead = p.Head.Head
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
ch.Members[p.Repository] = p.Head.Head
|
||||
for _, name := range p.Modules {
|
||||
fp, ok := p.Fps[name]
|
||||
if !ok || fp.Manifest == "" {
|
||||
if ok && fp.None2 == noModuleWords {
|
||||
ch.Removed = append(ch.Removed, name)
|
||||
continue
|
||||
}
|
||||
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
|
||||
}
|
||||
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
|
||||
ch.Rebuilt++
|
||||
}
|
||||
got := w.manifest(ctx, manifests, fp.Manifest)
|
||||
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
|
||||
return ch, got.err
|
||||
}
|
||||
ch.Digests[name] = fp.Manifest
|
||||
if got.err != "" {
|
||||
ch.Unreadable[name] = got.err
|
||||
continue
|
||||
}
|
||||
ch.Manifests[got.module] = got.manifest
|
||||
if definedIn[got.module] == nil {
|
||||
definedIn[got.module] = map[string]bool{}
|
||||
}
|
||||
definedIn[got.module][p.Repository] = true
|
||||
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
|
||||
definedIn[got.module][t.Repository] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
for name, repos := range definedIn {
|
||||
if len(repos) > 1 {
|
||||
for r := range repos {
|
||||
ch.Twice[name] = append(ch.Twice[name], r)
|
||||
}
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
return ch, ""
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
@@ -352,8 +287,7 @@ func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||
|
||||
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
|
||||
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
|
||||
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
|
||||
Reads: map[string][]string{}}
|
||||
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
@@ -365,14 +299,7 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNode {
|
||||
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
n.Reads[node] = h.Record.ManifestReads
|
||||
}
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(value); err == nil {
|
||||
@@ -406,10 +333,6 @@ func (w *world) value(bucket, key string) []byte {
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
|
||||
// manifest but the controller's reads the controller's grants.
|
||||
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
@@ -434,50 +357,36 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR
|
||||
// 0618 §1): nil when it is not done or names none.
|
||||
func (w *world) buildNeeds(module, fp string) []string {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(w.value(delivery.BuilderBuildsBucket, key))
|
||||
if err != nil || b.State != delivery.BuildDone {
|
||||
return nil
|
||||
}
|
||||
return b.Needs
|
||||
}
|
||||
|
||||
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
|
||||
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
|
||||
func (w *world) recordReads(node string) ([]string, bool) {
|
||||
r, ok := w.records()[node]
|
||||
return r.ManifestReads, ok
|
||||
}
|
||||
|
||||
// knownNodes is every node the assigner hears of: placed, with a record, or with a presence.
|
||||
func (w *world) knownNodes() []string {
|
||||
set := map[string]bool{}
|
||||
for node := range w.placedNodes() {
|
||||
set[node] = true
|
||||
}
|
||||
for node := range w.records() {
|
||||
set[node] = true
|
||||
}
|
||||
for node := range w.online() {
|
||||
set[node] = true
|
||||
}
|
||||
return slices.Sorted(maps.Keys(set))
|
||||
}
|
||||
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
if !w.placedNodes()[node] {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
@@ -492,14 +401,6 @@ func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// bucketGen is how many times any key of the bucket was written or deleted in this process: a read of what it shows
|
||||
// elsewhere is asked again only after it moves.
|
||||
func (w *world) bucketGen(bucket string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket]
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
@@ -507,35 +408,29 @@ func (w *world) changed(bucket, key string) uint64 {
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
raw, ok := w.read(delivery.HaveBucket)[key]
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil || h.Composed == "" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err == nil && isNode && !placed[node] {
|
||||
out = append(out, node)
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
|
||||
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
|
||||
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
|
||||
_, all := w.assigned()
|
||||
seen := map[string]bool{}
|
||||
for _, a := range all {
|
||||
if a.Module != module || a.Build == "" {
|
||||
continue
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, a.Build))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
|
||||
continue
|
||||
}
|
||||
seen[rec.Commit] = true
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for c := range seen {
|
||||
out = append(out, c)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
|
||||
@@ -2,12 +2,14 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// fakeManifests is the indexer's manifests bucket in memory.
|
||||
@@ -42,8 +44,76 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
|
||||
return p
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
|
||||
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
|
||||
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
|
||||
pull(assign.SDKRepository, 3, nil),
|
||||
}}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
|
||||
if waits != "" {
|
||||
t.Fatalf("the change waits: %s", waits)
|
||||
}
|
||||
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
|
||||
t.Errorf("change = %+v", ch)
|
||||
}
|
||||
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
|
||||
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
|
||||
// saying for what; an unreadable manifest is the change's, not a wait.
|
||||
func TestAChangeWaitsForItsManifests(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
|
||||
for _, c := range []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
|
||||
} {
|
||||
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
|
||||
if waits == "" {
|
||||
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
|
||||
}
|
||||
}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["web"] == "" {
|
||||
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
|
||||
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
|
||||
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
|
||||
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
|
||||
// xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"predefined"`) {
|
||||
t.Fatal("the fixture names no predefined key")
|
||||
}
|
||||
f := fakeManifests{}
|
||||
xterm := f.add(t, "xterm", string(raw))
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
@@ -54,27 +124,4 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
inPart bool
|
||||
granted bool
|
||||
}{
|
||||
{"the part's grant", true, true},
|
||||
{"no grant", false, false},
|
||||
} {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
|
||||
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
|
||||
}
|
||||
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
|
||||
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,32 +3,20 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
|
||||
|
||||
@@ -1,25 +1,23 @@
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639 h1:3FTxvHrv3Hv+ZUbp/ezTrndqdgiS+Z6EUG2XGYJNJr8=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:/V/1wfyQwjE9VbPLsFtkf6uX+SHKUXX34s5JscDuGAs=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639 h1:19PhtrCOwia711Oqdyafvs5EKTVmmgk5Gdb4leCkq3w=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:Dj6fbL8Evgy72EWO3Gl3L3Ra8N/nJSzgEVwSGmeRam0=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639 h1:YQ6eJvc+RYUVIVf+oX+R4TC1dYL8TLLMZ0/JPDU1tp4=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639/go.mod h1:TB20TsQuOd5aer+cXL6PbaYyg2RvgL1R4MzD61Oks3A=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639 h1:K/1yIkqAWKGtLEtJ0PibijIdpo5YATL/9IUVXFoEglM=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639/go.mod h1:osh0jQyNqYDM27tbxQo6HeUEXgLq9XUewKzNhUXadog=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -33,5 +31,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (delivery.AssignerAsk.Ask).
|
||||
func assignerAsk(asking int, a oc.Ask) delivery.AssignerAsk {
|
||||
raw, err := json.Marshal(a)
|
||||
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
|
||||
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
|
||||
}
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: raw}
|
||||
}
|
||||
|
||||
// AskOf is the router's ask an asks-state value carries.
|
||||
func AskOf(a delivery.AssignerAsk) (oc.Ask, error) {
|
||||
var ask oc.Ask
|
||||
if err := json.Unmarshal(a.Ask, &ask); err != nil {
|
||||
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
|
||||
}
|
||||
return ask, nil
|
||||
}
|
||||
|
||||
// EncodeAsk is delivery.EncodeAssignerAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
|
||||
// the ask's id is delivery.AskingID(key, asking), and it offers at least one option.
|
||||
func EncodeAsk(key string, a delivery.AssignerAsk) ([]byte, error) {
|
||||
if _, err := checkedAsk(key, a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return delivery.EncodeAssignerAsk(key, a)
|
||||
}
|
||||
|
||||
func checkedAsk(key string, a delivery.AssignerAsk) (oc.Ask, error) {
|
||||
ask, err := occ.Carried(a.Ask, delivery.AskingID(key, a.Asking))
|
||||
if err != nil {
|
||||
return oc.Ask{}, fmt.Errorf("assign: %w", err)
|
||||
}
|
||||
return ask, nil
|
||||
}
|
||||
|
||||
// DecodeAsk reads the ask at key (delivery.DecodeAssignerAsk) with the router's ask it carries.
|
||||
func DecodeAsk(key string, data []byte) (delivery.AssignerAsk, oc.Ask, error) {
|
||||
a, err := delivery.DecodeAssignerAsk(key, data)
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, oc.Ask{}, err
|
||||
}
|
||||
ask, err := checkedAsk(key, a)
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, oc.Ask{}, err
|
||||
}
|
||||
return a, ask, nil
|
||||
}
|
||||
|
||||
// ExpiresAt is when the ask ends unanswered; the zero time for an ask that names none.
|
||||
func ExpiresAt(a oc.Ask) time.Time {
|
||||
t, err := occ.Time(a.Expires)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -6,8 +6,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -16,21 +15,21 @@ import (
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Options: []oc.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(7 * 24 * time.Hour)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the version stays held",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
@@ -43,103 +42,69 @@ const (
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
||||
return assignerAsk(0, oc.Ask{
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
|
||||
Options: []asks.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(7 * 24 * time.Hour)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = occ.NeedsYou + c.Needs + ". " + what
|
||||
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > occ.HeadlineLength {
|
||||
headline = string([]rune(headline)[:occ.HeadlineLength])
|
||||
}
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > occ.HeadlineLength {
|
||||
headline = string([]rune(headline)[:occ.HeadlineLength])
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) oc.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) oc.Condition {
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
@@ -162,20 +127,20 @@ func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) del
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
||||
}[need.Judgement]
|
||||
return assignerAsk(0, oc.Ask{
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
||||
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(7 * 24 * time.Hour)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
||||
@@ -193,11 +158,7 @@ var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a delivery.AssignerAsk) string {
|
||||
ask, err := AskOf(a)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
|
||||
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
@@ -213,33 +174,33 @@ func BackupAnswerWords(option string, asking int) string {
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
|
||||
Options: []asks.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(7 * 24 * time.Hour)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module keeps running what it has",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
||||
// naming the node, the module and the backup's words; cleared under the same key.
|
||||
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
|
||||
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
||||
needs := "answer whether to back up again or go on without a backup"
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||
@@ -22,42 +18,27 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
}
|
||||
now := time.Now()
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
|
||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
if again.Asking != 7 || mustAsk(t, again).ID != delivery.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
|
||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||
}
|
||||
if _, err := EncodeAsk(key, again); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
|
||||
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
|
||||
good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0))
|
||||
other := good
|
||||
other.Asking = 3 // its id is the asking 0's
|
||||
empty := good
|
||||
ask := mustAsk(t, good)
|
||||
ask.Options = nil
|
||||
empty.Ask = assignerAsk(0, ask).Ask
|
||||
for name, a := range map[string]delivery.AssignerAsk{"another id": other, "no option": empty, "no ask": {}} {
|
||||
if _, err := EncodeAsk(key, a); err == nil {
|
||||
t.Errorf("%s: written", name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
@@ -69,102 +50,53 @@ func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range mustAsk(t, a).Options {
|
||||
for _, o := range a.Ask.Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation)
|
||||
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key {
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
// findings bucket cannot be written to at all.
|
||||
func saidConditionsAreFindings(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
raised, cleared oc.Condition
|
||||
key string
|
||||
}{
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
|
||||
f := Findings{Put: func(k string, v []byte) error {
|
||||
got, err := health.DecodeFinding(v)
|
||||
puts[k] = got
|
||||
return err
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(oc.Condition) error { said++; return nil })
|
||||
if err := say(c.raised); err != nil {
|
||||
t.Fatalf("%s: %v", c.key, err)
|
||||
}
|
||||
got, ok := puts[c.key]
|
||||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
|
||||
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
|
||||
}
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
|
||||
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
if err := (Findings{}).Say(func(oc.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
|
||||
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
|
||||
t.Error("a condition not the assigner's is a finding")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
@@ -181,24 +113,24 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("%s: the router refuses the ask: %v", key, err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("%s: cannot be written: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("%s: the view may not answer it: %v", key, err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", mustAsk(t, r))
|
||||
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", r.Ask)
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != delivery.BackupAgain || mustAsk(t, b).Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", mustAsk(t, b))
|
||||
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", b.Ask)
|
||||
}
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
@@ -211,7 +143,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
}
|
||||
c := BackupCondition("ace", "mail", fNew, "no space left", true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
@@ -223,39 +155,32 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := EncodeAsk(rkey, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
|
||||
yes := mustAsk(t, a).Options[0]
|
||||
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
|
||||
// acknowledge), as it did through the sdk before.
|
||||
if viewed == nil {
|
||||
if o, ok := occ.OptionOf(mustAsk(t, a), OptionYes); ok && o.Level != "" && o.Level != occ.Acknowledge {
|
||||
viewed = fmt.Errorf("only a channel that proves who answered takes the option %s", o.ID)
|
||||
}
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(mustAsk(t, a).Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, mustAsk(t, a).Explanation)
|
||||
case bus && (yes.Level != occ.Approve || yes.Binds == "" || viewed == nil || expiresAt(t, a).After(now.Add(occ.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, expiresAt(t, a))
|
||||
case !bus && (yes.Level != occ.Acknowledge || viewed != nil):
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if mustAsk(t, b).ID == mustAsk(t, a).ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, mustAsk(t, b).ID, mustAsk(t, a).ID)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
@@ -265,29 +190,14 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key {
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// mustAsk is the router's ask an asks-state value carries.
|
||||
func mustAsk(t *testing.T, a delivery.AssignerAsk) oc.Ask {
|
||||
t.Helper()
|
||||
ask, err := AskOf(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ask
|
||||
}
|
||||
|
||||
func expiresAt(t *testing.T, a delivery.AssignerAsk) time.Time {
|
||||
t.Helper()
|
||||
return ExpiresAt(mustAsk(t, a))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
|
||||
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
|
||||
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
|
||||
// and writes nothing itself: the program hands it what it watched and writes what it answers.
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
|
||||
const SDKRepository = "novox/mesh-sdk"
|
||||
|
||||
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
|
||||
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
|
||||
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
|
||||
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
|
||||
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
|
||||
"node-engine's validator would refuse"
|
||||
|
||||
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
|
||||
// why, and each node's open assignments.
|
||||
type Mesh struct {
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Assigned map[string][]string
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
|
||||
// member (empty when none), and how many modules it rebuilds.
|
||||
type Change struct {
|
||||
Group, Heads string
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Removed []string
|
||||
Twice map[string][]string
|
||||
SDKHead string
|
||||
Rebuilt int
|
||||
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine.
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
moved[name] = true
|
||||
}
|
||||
for name := range c.Unreadable {
|
||||
moved[name] = true
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
moved[name] = true
|
||||
}
|
||||
provided, required := map[string]bool{}, map[string]bool{}
|
||||
for name := range moved {
|
||||
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
|
||||
for _, o := range man.Provides {
|
||||
provided[o.Name] = true
|
||||
}
|
||||
}
|
||||
for _, r := range c.Manifests[name].Requires {
|
||||
required[r] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
|
||||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
|
||||
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
|
||||
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
|
||||
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
|
||||
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
|
||||
in := compose.Input{Catalogue: catalog}
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
for _, mod := range assigned[n] {
|
||||
for _, cl := range catalog[mod].Claims {
|
||||
scope := cl.Scope
|
||||
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||
scope = seat.Scope
|
||||
}
|
||||
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
|
||||
}
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
|
||||
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
|
||||
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
|
||||
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
|
||||
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
|
||||
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
|
||||
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
|
||||
var fails []string
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Unreadable) {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
trunk := input(m.Manifests, m.Assigned)
|
||||
changed := map[string]catalogue.Manifest{}
|
||||
for name, man := range m.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for name, man := range c.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
switch {
|
||||
case now == nil:
|
||||
case before != nil:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
|
||||
}
|
||||
}
|
||||
if c.SDKHead != "" {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
|
||||
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
|
||||
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
|
||||
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
|
||||
if c.Rebuilt > delivery.CompositionRebuildWarning {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
|
||||
}
|
||||
v.Warnings = append(v.Warnings, NotJudged)
|
||||
if could := CouldRun(m, c, machines); len(could) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
|
||||
}
|
||||
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
|
||||
}
|
||||
if len(fails) > 0 {
|
||||
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
|
||||
return v
|
||||
}
|
||||
v.State = delivery.CompositionPass
|
||||
return v
|
||||
}
|
||||
|
||||
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
|
||||
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
|
||||
// the facts snapshot), so each is named as not judged.
|
||||
func CouldRun(m Mesh, c Change, composed []string) []string {
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for name := range c.Manifests {
|
||||
if !slices.Contains(modules, name) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
|
||||
func grantsOf(man catalogue.Manifest) string {
|
||||
var offers []string
|
||||
for _, o := range man.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
|
||||
}
|
||||
|
||||
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
|
||||
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
|
||||
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
|
||||
func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
provisions := map[string]bool{}
|
||||
altered := false
|
||||
for name, man := range c.Manifests {
|
||||
before, had := m.Manifests[name]
|
||||
if had && grantsOf(before) == grantsOf(man) {
|
||||
continue
|
||||
}
|
||||
altered = true
|
||||
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
|
||||
provisions[r] = true
|
||||
}
|
||||
}
|
||||
if !altered {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func meshOf() Mesh {
|
||||
return Mesh{
|
||||
Manifests: map[string]catalogue.Manifest{
|
||||
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
|
||||
"web": {Module: "web", Version: "1"},
|
||||
},
|
||||
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
|
||||
}
|
||||
}
|
||||
|
||||
func hasWarning(v delivery.CompositionVerdict, part string) bool {
|
||||
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
|
||||
}
|
||||
|
||||
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
|
||||
// the machine and the mesh-sdk it composed with, and says what it did not judge.
|
||||
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "v0.1.14-x", 0)
|
||||
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
|
||||
}
|
||||
if !hasWarning(v, "issue 1461") {
|
||||
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||
// consumer's machine is touched when its provider changes.
|
||||
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
|
||||
func TestUnreadableTwiceAndWide(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
|
||||
v = Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
|
||||
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
|
||||
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
|
||||
v := Compose(meshOf(), c, "v0.1.14-own", 0)
|
||||
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
|
||||
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
|
||||
}
|
||||
c.SDKHead = ""
|
||||
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
|
||||
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
|
||||
// assigned nowhere touches none, and the verdict says so.
|
||||
func TestTouched(t *testing.T) {
|
||||
m := meshOf()
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
|
||||
t.Errorf("touched by store = %v", got)
|
||||
}
|
||||
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
|
||||
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
|
||||
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
|
||||
// names none of the second.
|
||||
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
|
||||
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
|
||||
m.Assigned["novox"] = []string{"bus"}
|
||||
m.Assigned["shanks"] = []string{"db"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
|
||||
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
|
||||
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
|
||||
}
|
||||
if !hasWarning(v, "whose grants alone the change alters: novox") {
|
||||
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
|
||||
}
|
||||
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
|
||||
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
|
||||
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
|
||||
const (
|
||||
CaseTrunk = "trunk"
|
||||
CaseChange = "change"
|
||||
)
|
||||
|
||||
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
|
||||
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
|
||||
const SDKDir = "go"
|
||||
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
|
||||
// them, the node-engine applies them. Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
|
||||
|
||||
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
|
||||
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
|
||||
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
|
||||
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
|
||||
for name, digest := range m.Digests {
|
||||
trunk[name] = composer.Manifest{Digest: digest}
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for name, digest := range c.Digests {
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: true, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
|
||||
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
|
||||
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
|
||||
changed := map[string]composer.Manifest{}
|
||||
for name, digest := range c.Digests {
|
||||
changed[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
|
||||
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
|
||||
}
|
||||
|
||||
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return delivery.ComposeJob{}, err
|
||||
}
|
||||
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
|
||||
Package: composer.Package, Request: raw, Reader: reader}, nil
|
||||
}
|
||||
|
||||
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
|
||||
// composer derives the seats held from the claims, as input does).
|
||||
func plainInput(assigned map[string][]string) compose.Input {
|
||||
var in compose.Input
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
|
||||
type Ran struct {
|
||||
State string
|
||||
Reason string
|
||||
Verdict *composer.Verdict
|
||||
}
|
||||
|
||||
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
|
||||
// failure, named.
|
||||
func RanOf(step delivery.Step) Ran {
|
||||
r := Ran{State: step.State, Reason: step.Reason}
|
||||
if step.State != delivery.StepPassed {
|
||||
return r
|
||||
}
|
||||
var a delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &a); err != nil {
|
||||
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
|
||||
}
|
||||
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
|
||||
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
|
||||
}
|
||||
r.Verdict = &v
|
||||
return r
|
||||
}
|
||||
|
||||
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
|
||||
if v == nil {
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name == name {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
|
||||
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
|
||||
// the group holds one in its repository, else a build its node runs.
|
||||
type Reader struct {
|
||||
Module, Commit string
|
||||
Member bool
|
||||
Ran Ran
|
||||
}
|
||||
|
||||
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
|
||||
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
|
||||
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
|
||||
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
|
||||
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
|
||||
// said and judged nothing.
|
||||
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
var fails []string
|
||||
if head.State != delivery.StepPassed {
|
||||
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
|
||||
v.State = delivery.CompositionFail
|
||||
return v
|
||||
}
|
||||
if head.Verdict.SDK != "" {
|
||||
v.SDK = head.Verdict.SDK
|
||||
}
|
||||
trunk, okT := caseOf(head.Verdict, CaseTrunk)
|
||||
change, okC := caseOf(head.Verdict, CaseChange)
|
||||
if !okT || !okC {
|
||||
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
|
||||
return v
|
||||
}
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if r := change.Manifests[name]; !r.Accepted {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(trunk.Manifests) {
|
||||
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
if _, before := m.Unreadable[name]; before {
|
||||
continue // already unreadable on the trunk: not the group's doing
|
||||
}
|
||||
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
before, now := trunk.Machines[node], change.Machines[node]
|
||||
switch {
|
||||
case now.Composes:
|
||||
case !before.Composes:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
|
||||
}
|
||||
}
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
|
||||
// from before issue 1621): Go's words for a package its module does not hold.
|
||||
func NoComposer(reason string) bool {
|
||||
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
|
||||
if strings.Contains(reason, words) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
|
||||
for _, r := range readers {
|
||||
at := r.Module + " at " + r.Commit
|
||||
if r.Member {
|
||||
at = r.Module + "'s member head " + r.Commit
|
||||
}
|
||||
if r.Ran.State != delivery.StepPassed {
|
||||
if NoComposer(r.Ran.Reason) {
|
||||
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
|
||||
"composer command (before issue 1621)", at))
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
|
||||
continue
|
||||
}
|
||||
read, ok := caseOf(r.Ran.Verdict, CaseChange)
|
||||
if !ok {
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
|
||||
continue
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
|
||||
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
|
||||
}
|
||||
}
|
||||
return fails, warnings
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
const sdkHead = "1111111111111111111111111111111111111111"
|
||||
|
||||
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
|
||||
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
|
||||
var raws = map[string]string{
|
||||
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
|
||||
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
|
||||
"d-web": `{"module": "web", "version": "1"}`,
|
||||
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
|
||||
}
|
||||
|
||||
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
|
||||
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
|
||||
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
|
||||
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
|
||||
|
||||
func digestedMesh() Mesh {
|
||||
m := meshOf()
|
||||
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
|
||||
return m
|
||||
}
|
||||
|
||||
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
|
||||
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
|
||||
func webShares(members map[string]string) Change {
|
||||
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
|
||||
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
|
||||
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
|
||||
}
|
||||
|
||||
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
|
||||
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
|
||||
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
|
||||
t.Helper()
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := composer.Fill(&req, func(d string) ([]byte, error) {
|
||||
if r, ok := read[d]; ok {
|
||||
return []byte(r), nil
|
||||
}
|
||||
return []byte(raws[d]), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.SDK = "(devel) " + job.Fetch["commit"].(string)
|
||||
vr, _ := json.Marshal(v)
|
||||
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
|
||||
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
|
||||
}
|
||||
|
||||
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
|
||||
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
|
||||
// change by digest, never the manifests' bytes, composing the machines the change touches.
|
||||
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, err := HeadJob(digestedMesh(), c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
|
||||
t.Fatalf("job = %+v", job)
|
||||
}
|
||||
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: testTime, Compose: &job}
|
||||
if _, err := delivery.EncodeStepAsk(ask); err != nil {
|
||||
t.Fatalf("the ask is refused by its layout: %v", err)
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
|
||||
t.Fatalf("cases = %+v", req.Cases)
|
||||
}
|
||||
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
|
||||
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
|
||||
}
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
|
||||
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
|
||||
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
v := FromComposer(digestedMesh(), c, head, nil, 0)
|
||||
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
|
||||
t.Fatalf("with the head's composer: %+v", v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
|
||||
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
|
||||
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
|
||||
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
|
||||
// the SDK head (its composer reads what the head reads) it passes.
|
||||
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
running := "3333333333333333333333333333333333333333"
|
||||
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
|
||||
t.Fatalf("reader job = %+v", rjob)
|
||||
}
|
||||
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
|
||||
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
|
||||
}
|
||||
member := "2222222222222222222222222222222222222222"
|
||||
pinned := run(t, rjob, asTheHeadReads)
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
|
||||
if v.State != delivery.CompositionPass {
|
||||
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
|
||||
// failed for another cause fails the group, loud.
|
||||
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
|
||||
t.Fatalf("a reader with no composer: %+v", v)
|
||||
}
|
||||
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
|
||||
t.Fatalf("a reader whose composer failed: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
|
||||
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
|
||||
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
|
||||
t.Fatalf("a head composer that did not run: %+v", v)
|
||||
}
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
|
||||
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
|
||||
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
|
||||
// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair
|
||||
// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its
|
||||
// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router
|
||||
// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the
|
||||
// assigner no longer says it straight (never both).
|
||||
|
||||
// FindingsBucket is the assigner's findings.
|
||||
var FindingsBucket = health.FindingsBucket(delivery.AssignerModule)
|
||||
|
||||
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
|
||||
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
|
||||
// a clear, whose key is to be deleted; value is nil then.
|
||||
func FindingOf(c oc.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
|
||||
rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".")
|
||||
if !ok || rest == "" {
|
||||
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
|
||||
}
|
||||
kind, id, _ := strings.Cut(rest, ".")
|
||||
if id == "" {
|
||||
id = c.Subject.ID
|
||||
}
|
||||
scope := c.Subject.Scope
|
||||
if scope == "" {
|
||||
scope = health.ScopeModule
|
||||
}
|
||||
key = health.FindingKey(scope, id, kind)
|
||||
if c.State == occ.StateCleared {
|
||||
return key, nil, false, nil
|
||||
}
|
||||
if c.State != occ.StateRaised {
|
||||
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
|
||||
}
|
||||
summary := c.Summary
|
||||
if summary == "" {
|
||||
summary = c.Headline
|
||||
}
|
||||
var actions []health.Action
|
||||
for _, a := range c.Actions {
|
||||
actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
|
||||
}
|
||||
value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine},
|
||||
Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved,
|
||||
Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind,
|
||||
Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
return key, value, true, err
|
||||
}
|
||||
|
||||
// Findings is where the assigner's findings go: the bucket's put and delete.
|
||||
type Findings struct {
|
||||
Put func(key string, value []byte) error
|
||||
Delete func(key string) error
|
||||
Keys func() ([]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
|
||||
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
|
||||
// findings bucket at all. A finding that cannot be written is the error.
|
||||
func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
|
||||
return func(c oc.Condition) error {
|
||||
var errs []error
|
||||
if f.Put != nil {
|
||||
now := time.Now
|
||||
if f.Now != nil {
|
||||
now = f.Now
|
||||
}
|
||||
key, value, raised, err := FindingOf(c, now())
|
||||
switch {
|
||||
case err != nil:
|
||||
errs = append(errs, err)
|
||||
case raised:
|
||||
if err := f.Put(key, value); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
|
||||
}
|
||||
default:
|
||||
if err := f.Delete(key); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.Put == nil {
|
||||
if say != nil {
|
||||
if err := say(c); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) oc.Condition {
|
||||
key := delivery.AssignerModule + ".seat-states"
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: delivery.AssignerModule}
|
||||
if len(unbound) == 0 {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: occ.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
|
||||
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
|
||||
|
||||
// PoliciesBucket is the assigner's state `policies`.
|
||||
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
|
||||
|
||||
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
|
||||
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
|
||||
out := map[string]catalogue.PersonPolicy{}
|
||||
var bad []string
|
||||
for key, raw := range values {
|
||||
var p catalogue.PersonPolicy
|
||||
if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key {
|
||||
bad = append(bad, key)
|
||||
continue
|
||||
}
|
||||
out[key] = p
|
||||
}
|
||||
sort.Strings(bad)
|
||||
return out, bad
|
||||
}
|
||||
|
||||
// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one.
|
||||
func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) {
|
||||
var c *catalogue.PersonPolicy
|
||||
if p, ok := chosen[m.Module]; ok {
|
||||
c = &p
|
||||
}
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
|
||||
const KindProvisionFailed = "provision-failed"
|
||||
|
||||
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
|
||||
func ProvisionFailedKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed)
|
||||
}
|
||||
|
||||
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
|
||||
func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
|
||||
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
|
||||
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
|
||||
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}},
|
||||
Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func also(provider, consumer string) []string {
|
||||
if provider == "" || provider == consumer {
|
||||
return nil
|
||||
}
|
||||
return []string{provider}
|
||||
}
|
||||
|
||||
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
|
||||
const KindProvisionMissing = "provision-missing"
|
||||
|
||||
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
|
||||
func ProvisionMissingKey(key string) string {
|
||||
return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
|
||||
}
|
||||
|
||||
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
|
||||
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
|
||||
parts := strings.SplitN(key, ".", 3)
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
|
||||
}
|
||||
node, module, provision := parts[0], parts[1], parts[2]
|
||||
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
|
||||
"no provider of it may be placed", module, node, provision, bound, module)
|
||||
return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
|
||||
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
|
||||
Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
|
||||
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
|
||||
Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6).
|
||||
const KindUnplaced = "unplaced"
|
||||
|
||||
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
|
||||
func UnplacedKey(node string) string {
|
||||
return health.FindingKey(health.ScopeMachine, node, KindUnplaced)
|
||||
}
|
||||
|
||||
// UnplacedFinding is the finding of a node no placement names since the lease write.
|
||||
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+
|
||||
"so the assigner writes it no want", node)
|
||||
return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced,
|
||||
Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary,
|
||||
Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.",
|
||||
Explanation: occ.NeedsYou + "place it with mesh-placement.propose. " + summary + ".",
|
||||
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func clipHeadline(s string) string {
|
||||
if len([]rune(s)) <= 60 {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:60])
|
||||
}
|
||||
@@ -4,8 +4,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -69,7 +68,7 @@ func StepPut(w PutWorld) PutNext {
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
@@ -211,15 +210,15 @@ func PutAsking(p delivery.Put) int {
|
||||
}
|
||||
|
||||
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||
func PutAct(node, module, fp, composed string) occ.Act {
|
||||
return occ.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
func PutAct(node, module, fp, composed string) asks.Act {
|
||||
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
}
|
||||
|
||||
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||
yes := oc.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: occ.Acknowledge}
|
||||
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
@@ -228,26 +227,26 @@ func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest,
|
||||
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||
}[d.Judgement]
|
||||
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||
expires := occ.FormatTime(now.Add(7 * 24 * time.Hour))
|
||||
expires := now.Add(7 * 24 * time.Hour)
|
||||
if bus {
|
||||
binds, err := occ.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
yes.Level, yes.Binds = occ.Approve, binds
|
||||
yes.Level, yes.Binds = asks.Approve, binds
|
||||
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||
expires = occ.FormatTime(now.Add(occ.ApproveLasts))
|
||||
expires = now.Add(asks.ApproveLasts)
|
||||
}
|
||||
return assignerAsk(PutAsking(p), oc.Ask{
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, PutAsking(p)),
|
||||
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||
Explanation: explanation,
|
||||
Options: []oc.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: occ.Acknowledge}},
|
||||
Who: occ.Operator,
|
||||
Options: []asks.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||
Who: asks.Operator,
|
||||
Expires: expires,
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}), nil
|
||||
}, Asking: PutAsking(p)}, nil
|
||||
}
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
|
||||
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
|
||||
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
|
||||
|
||||
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
|
||||
Warrant *delivery.AddNodeWarrant
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
}
|
||||
if p.Why == "" {
|
||||
findings = append(findings, "a proposal says why: it is shown to the operator")
|
||||
}
|
||||
if len(in.Changes) == 0 {
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
}
|
||||
out := map[string]*delivery.Placement{}
|
||||
for node, pl := range laid {
|
||||
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
|
||||
continue
|
||||
}
|
||||
pl := pl
|
||||
all[node] = pl
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
|
||||
// candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
|
||||
for _, node := range p.RemovedNodes() {
|
||||
if _, ok := in.Current[node]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(all, node)
|
||||
out[node] = nil
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
held, recorded := in.Held, in.Recorded
|
||||
if held == nil {
|
||||
held = func(string) bool { return false }
|
||||
}
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
// A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by
|
||||
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
|
||||
joining := map[string]bool{}
|
||||
for _, c := range in.Changes {
|
||||
if c.Op == delivery.OpAddNode {
|
||||
joining[c.Node] = true
|
||||
}
|
||||
}
|
||||
if len(joining) > 0 {
|
||||
was := recorded
|
||||
recorded = func(node string) bool { return joining[node] || was(node) }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// ControllerModule is the controller's module, which nothing places again (issue 1126).
|
||||
const ControllerModule = "mesh-controller"
|
||||
|
||||
// ControllerEnded is why a placement of the controller is refused: the parts that acted only once its lease was handed
|
||||
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
|
||||
// would write beside them, silently.
|
||||
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
|
||||
|
||||
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
|
||||
// whole placement, or in what the changes lay. None when nothing does.
|
||||
func ControllerRefusals(changes []delivery.PlacementProposalChange, laid map[string]delivery.Placement) []string {
|
||||
var out []string
|
||||
nodes := map[string]bool{}
|
||||
for i, c := range changes {
|
||||
places := c.Module == ControllerModule && c.Op != delivery.OpTakeOff
|
||||
if c.Whole != nil && slices.Contains(c.Whole.Assigned(), ControllerModule) {
|
||||
places = true
|
||||
}
|
||||
if places {
|
||||
out = append(out, fmt.Sprintf("change %d places %s on %s: %s", i+1, ControllerModule, c.Node, ControllerEnded))
|
||||
nodes[c.Node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(laid)) {
|
||||
if !nodes[node] && slices.Contains(laid[node].Assigned(), ControllerModule) {
|
||||
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
|
||||
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
|
||||
// it is created) and its value (nil: it is deleted).
|
||||
type Write struct {
|
||||
Node, Key string
|
||||
Base uint64
|
||||
Value *delivery.Placement
|
||||
}
|
||||
|
||||
// Writes is every key a proposal writes, by node order.
|
||||
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
|
||||
var out []Write
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
|
||||
// the controller's verbs (ADR 0567 §8, issue 774).
|
||||
const (
|
||||
KindControllerCall = "controller-call"
|
||||
KindControllerPin = "controller-pin"
|
||||
)
|
||||
|
||||
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
|
||||
// module, or node and provision for a pin).
|
||||
func CallClearedCondition(kind, node, module string) oc.Condition {
|
||||
return oc.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
|
||||
Subject: oc.ConditionSubject{Scope: "module", ID: module, Machine: node}, State: occ.StateCleared,
|
||||
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
func PlacementAct(p delivery.PlacementProposal) occ.Act {
|
||||
return occ.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
|
||||
}
|
||||
|
||||
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
|
||||
// another asking, under an id of its own.
|
||||
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
|
||||
return int(at.Unix())
|
||||
}
|
||||
|
||||
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
|
||||
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
|
||||
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
|
||||
binds, err := occ.ActDigest(PlacementAct(p))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if n := len([]rune(headline)); n > occ.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: headline,
|
||||
Explanation: brief,
|
||||
Options: []oc.Option{
|
||||
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
|
||||
Level: occ.Approve, Binds: binds},
|
||||
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: occ.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(occ.ApproveLasts)),
|
||||
OnExpiry: "nothing is written, and the proposal ends",
|
||||
About: key,
|
||||
}), nil
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
rest := p
|
||||
rest.Changes, rest.Base = nil, map[string]uint64{}
|
||||
for _, c := range p.Changes {
|
||||
if _, done := written[c.Node]; !done {
|
||||
rest.Changes = append(rest.Changes, c)
|
||||
}
|
||||
}
|
||||
for node, base := range p.Base {
|
||||
if _, done := written[node]; !done {
|
||||
rest.Base[node] = base
|
||||
}
|
||||
}
|
||||
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
|
||||
return rest, len(rest.Changes) > 0
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
|
||||
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
|
||||
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
|
||||
func TestThePlacementSeatDecides(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
|
||||
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
|
||||
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
|
||||
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
|
||||
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
|
||||
Held: func(m string) bool { return m != "ghost" },
|
||||
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
|
||||
}
|
||||
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
|
||||
p, laid, findings := Propose(in(move...))
|
||||
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
|
||||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
|
||||
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
|
||||
}
|
||||
if _, err := delivery.EncodePlacementProposal(p); err != nil {
|
||||
t.Fatalf("the proposal cannot be written: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
|
||||
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
|
||||
// act without its lease.
|
||||
"the controller placed again": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
|
||||
"the controller seeded whole": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "anchor", Whole: &delivery.Placement{V: delivery.Version, Node: "anchor",
|
||||
Decided: anchor.Decided, Modules: []delivery.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("%s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
|
||||
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
|
||||
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
|
||||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
|
||||
t.Errorf("the rest = %+v, %v", rest, ok)
|
||||
}
|
||||
if _, ok := Rest(p, laid); ok {
|
||||
t.Error("a proposal written whole leaves something to write")
|
||||
}
|
||||
|
||||
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
|
||||
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
|
||||
c := SeatWaitsCondition(unbound)
|
||||
if _, err := occ.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == occ.StateRaised) ||
|
||||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
|
||||
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
|
||||
}
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || occ.Check(mustAsk(t, a), time.Unix(100, 0)) != nil || mustAsk(t, a).Options[0].Level != occ.Approve || occ.Performs(mustAsk(t, a).Options[0], PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
q := p
|
||||
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
|
||||
if occ.Performs(mustAsk(t, a).Options[0], PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
|
||||
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
|
||||
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
|
||||
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
|
||||
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
|
||||
func (m Module) verbWaiting(node string) (string, bool) {
|
||||
if m.Served == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
|
||||
if !m.Served(v) {
|
||||
return graph.VerbWaitWords(v, m.Name, node), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
|
||||
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
|
||||
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
|
||||
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
|
||||
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
|
||||
func (m Module) busStateWaiting(node string) (string, bool) {
|
||||
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
|
||||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
|
||||
|
||||
return "", false
|
||||
}
|
||||
return graph.BusStateWaitWords(m.Name, node), true
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -10,10 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/forms"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||
@@ -33,40 +28,13 @@ type Module struct {
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
|
||||
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
|
||||
Provisions func(node string) map[string]provisions.Provision
|
||||
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
||||
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
||||
Answering func(provision string) []catalogue.Manifest
|
||||
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
|
||||
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
|
||||
// is waited on for want of an offer.
|
||||
Offered func(provision string) bool
|
||||
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
|
||||
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
|
||||
// served (Served). Nil: no graph node is read, and no want waits on a verb.
|
||||
Invokes []string
|
||||
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
|
||||
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
|
||||
Served func(invoke string) bool
|
||||
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
|
||||
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
|
||||
// runs, and no want waits on bus state.
|
||||
Offering func(provision string) bool
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
|
||||
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
|
||||
// approved release is known to carry it. Nil: none is read.
|
||||
Release func(fp string) (string, bool)
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
@@ -78,21 +46,8 @@ type Module struct {
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
|
||||
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
|
||||
Needs []string
|
||||
// FormNeeds are the reference forms the module's build at FP is written in (`form:<name>`, the builder's build key's
|
||||
// needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the
|
||||
// build names none, and holds nothing.
|
||||
FormNeeds []string
|
||||
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
||||
Self func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
@@ -106,9 +61,6 @@ type Nodes struct {
|
||||
Haves map[string]map[string]delivery.Have
|
||||
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
|
||||
Providers map[string]map[string][]string
|
||||
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
|
||||
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
|
||||
Reads map[string][]string
|
||||
}
|
||||
|
||||
// healthy says whether a have runs fp healthy.
|
||||
@@ -130,15 +82,14 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
|
||||
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
|
||||
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
|
||||
// down and one module never blocks another.
|
||||
// limit (ADR 0528 clause 3).
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
|
||||
// without which there is nothing to want: the build done, not held, the grant and the composition.
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
@@ -154,6 +105,11 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
@@ -162,114 +118,12 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if words, lacking := m.readerLacking(node, n); lacking {
|
||||
return false, delivery.RuleComposed, words
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.verbWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.busStateWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
|
||||
// manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the
|
||||
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
|
||||
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
|
||||
//
|
||||
// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a
|
||||
// first form, so a build written in one is never held by its forms; the words name the forms it reads none of.
|
||||
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
|
||||
missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node])
|
||||
if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) {
|
||||
missing = slices.Clone(m.FormNeeds)
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
|
||||
}
|
||||
|
||||
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
||||
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
||||
func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
if m.Provisions == nil {
|
||||
return "", false
|
||||
}
|
||||
if missing := m.MissingProvisions(node); len(missing) > 0 {
|
||||
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
||||
}
|
||||
keyed := m.Provisions(node)
|
||||
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
|
||||
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
|
||||
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
|
||||
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
|
||||
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
|
||||
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
|
||||
// bound (ADR 0603 §8).
|
||||
needed := m.Manifest.Requires
|
||||
if m.Answering != nil {
|
||||
needed = catalogue.Awaited(m.Manifest, m.Answering)
|
||||
have := map[string]bool{}
|
||||
for _, p := range keyed {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
|
||||
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
|
||||
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
|
||||
continue
|
||||
}
|
||||
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
|
||||
continue
|
||||
}
|
||||
needed = append(needed, r)
|
||||
if !have[r] {
|
||||
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
||||
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
|
||||
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
||||
if p.Why != "" {
|
||||
words += ": " + p.Why
|
||||
}
|
||||
return words, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
|
||||
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
|
||||
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
|
||||
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
|
||||
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
|
||||
func (m Module) MissingProvisions(node string) []string {
|
||||
if m.Provisions == nil || m.Answering == nil {
|
||||
return nil
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, p := range m.Provisions(node) {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
var out []string
|
||||
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
|
||||
if !have[r] {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
||||
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
||||
// offers it) and holds.
|
||||
@@ -296,17 +150,6 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
||||
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
||||
}
|
||||
|
||||
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
|
||||
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
|
||||
func unreached(m Module, node string, n Nodes) string {
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return why
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
||||
// valuable or rebuildable; cache and none do not).
|
||||
func keepsData(m catalogue.Manifest) bool {
|
||||
@@ -326,9 +169,6 @@ type Outcome struct {
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
||||
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
||||
NodeFailed map[string]NodeFailure
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
@@ -336,9 +176,6 @@ type Outcome struct {
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
|
||||
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
|
||||
Unreached map[string]string
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
@@ -355,18 +192,12 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
|
||||
// §2): no bar of its own.
|
||||
// Holding is a rule holding a node back, and why.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words.
|
||||
func (h Holding) Reason() string {
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
@@ -380,25 +211,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || h.FP != m.FP || !failed(h) {
|
||||
continue
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
||||
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
||||
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
||||
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
||||
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
||||
// §2, ADR 0429 §6).
|
||||
if out.NodeFailed == nil {
|
||||
out.NodeFailed = map[string]NodeFailure{}
|
||||
}
|
||||
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
||||
continue
|
||||
}
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
}
|
||||
mm := m
|
||||
@@ -443,41 +260,27 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
|
||||
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
|
||||
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
|
||||
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
|
||||
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
|
||||
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
|
||||
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
|
||||
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
if older {
|
||||
_, composed := m.composition(node, have.FP)
|
||||
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
|
||||
}
|
||||
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
||||
switch {
|
||||
case older:
|
||||
ok, target = true, have.FP
|
||||
case oldForm:
|
||||
// held back by rule 3 as written: on neither list until the canary runs F healthy
|
||||
default:
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
}
|
||||
}
|
||||
if why := unreached(m, node, n); why != "" {
|
||||
if out.Unreached == nil {
|
||||
out.Unreached = map[string]string{}
|
||||
}
|
||||
out.Unreached[node] = why
|
||||
}
|
||||
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
||||
n.Haves[node][m.Name].FP != m.FP {
|
||||
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
||||
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
||||
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
||||
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
d, _ := m.composition(node, target)
|
||||
@@ -488,8 +291,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -566,10 +368,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
}
|
||||
}
|
||||
next, write := m.selfDerived(node, before, had, next, n, &out)
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
@@ -582,43 +380,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
|
||||
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
||||
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
||||
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
||||
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
||||
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
||||
// while its ask waits, the want written stays and the node's holding says why.
|
||||
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
|
||||
switch {
|
||||
case next.Removed:
|
||||
return next, true
|
||||
case m.selfOn(node):
|
||||
next.Self, next.Composed = true, ""
|
||||
return next, true
|
||||
case !next.Self && next.Composed != "":
|
||||
return next, true
|
||||
}
|
||||
next.Self, next.Yes = false, ""
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
at.FP = next.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
return next, false
|
||||
}
|
||||
next.Composed = d.Digest
|
||||
return next, true
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
@@ -633,43 +394,6 @@ func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
||||
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
||||
var out []string
|
||||
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
||||
// healthy now.
|
||||
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
||||
out = append(out, other)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
||||
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
||||
func failedElsewhere(m Module, n Nodes, node string) bool {
|
||||
for _, other := range m.Nodes {
|
||||
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
||||
func FailedAloneWords(module, fp string, failing []string) string {
|
||||
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
||||
strings.Join(failing, ", "))
|
||||
}
|
||||
|
||||
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
||||
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
||||
type NodeFailure struct {
|
||||
Others []string
|
||||
Words string
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
@@ -727,10 +451,10 @@ func short(fp string) string {
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
|
||||
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
|
||||
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
|
||||
// streams before it applies (§2); the assigner asks no snapshot for it.
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
@@ -742,6 +466,14 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
@@ -751,14 +483,9 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
|
||||
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
||||
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if m.Chosen != nil {
|
||||
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
@@ -788,8 +515,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.Grant, node, module string) bool {
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -8,9 +8,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
@@ -68,16 +65,13 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
delivery.RuleCanary: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
},
|
||||
delivery.RuleProviders: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
},
|
||||
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
||||
},
|
||||
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
|
||||
delivery.RuleProvisions: func(m *Module, _ *Nodes) {
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
|
||||
}
|
||||
},
|
||||
} {
|
||||
m, n := ready()
|
||||
breakIt(&m, &n)
|
||||
@@ -91,346 +85,12 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
|
||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||
}
|
||||
if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
|
||||
t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
|
||||
for _, all := range []func(string) map[string]provisions.Provision{
|
||||
func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}}
|
||||
}, nil} {
|
||||
m, n := ready()
|
||||
m.Provisions = all
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew {
|
||||
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §4: a want is held on a node whose engine does not state a reader capability the manifest needs, under rule 7
|
||||
// with the capability as its words, and moves once the engine states it; an engine stating nothing reads the baseline,
|
||||
// so a manifest of the baseline's words, the node-engine's own new build among them, is not held.
|
||||
hostNeeds, err := catalogue.ManifestNeeds([]byte(`{"module":"mesh-host","version":"1","slug":"host","build":{"artifacts":[` +
|
||||
`{"name":"host-arch","kind":"bundle","language":"go","system":"arch","from":"cmd/mesh-host"}]},"consumes":["restic.backed-up"],` +
|
||||
`"state":[{"name":"health"}],"config":{"x":{"type":"number","minimum":1,"default":50,"runtime":true}},` +
|
||||
`"resources":[{"id":"k","type":"file","path":"/var/lib/k","content":"${config:x}\n"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
name string
|
||||
needs []string
|
||||
reads map[string][]string
|
||||
held bool
|
||||
// forms are the build's reference forms (ADR 0624 §2), and names what the hold's words name.
|
||||
forms []string
|
||||
names string
|
||||
}{
|
||||
{"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true, nil, "word:tool-runner"},
|
||||
{"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"},
|
||||
map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false, nil, ""},
|
||||
{"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false, nil, ""},
|
||||
{"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false, nil, ""},
|
||||
// ADR 0624 §2, with form:new a form no engine reads unless it states it and artifact-store the first form.
|
||||
{"g14 back stating {old}, a build carrying {old,new}: moves", "mail", nil, map[string][]string{"g14": {"word:module"}}, false,
|
||||
[]string{"form:artifact-store", "form:new"}, ""},
|
||||
{"an engine stating nothing, a build carrying the baseline's form: moves", "mail", nil, nil, false, []string{"form:artifact-store"}, ""},
|
||||
{"a build carrying {new} only: held, naming the form", "mail", nil, map[string][]string{"g14": {"word:module"}}, true,
|
||||
[]string{"form:new"}, "form:new"},
|
||||
{"the same, once g14's engine states it", "mail", nil, map[string][]string{"g14": {"form:new"}}, false, []string{"form:new"}, ""},
|
||||
{"a manifest needing an unstated word, its build in the first form: held, naming the word", "mail", []string{"word:tool-runner"}, nil, true,
|
||||
[]string{"form:artifact-store"}, "word:tool-runner"},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms
|
||||
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if moved := out.Wants["g14"].FP == fNew; moved == c.held {
|
||||
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
if r := out.Rules["g14"]; c.held && (r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, c.names) ||
|
||||
!strings.Contains(r.Words, "node-engine of g14")) {
|
||||
t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r)
|
||||
}
|
||||
if w := out.Wants["g14"]; c.held && (w.Rule != delivery.RuleComposed || w.Newest != fNew) {
|
||||
t.Errorf("%s: g14's want %+v, want it kept at what it runs and naming rule 7", c.why, w)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §2, §3: a want waits, under the provisions' rule, until each verb it invokes is served by a running build,
|
||||
// and moves once one is; a verb a running build serves already adds no wait, and an edge the graph leaves out on a cycle
|
||||
// is not among Invokes, so it holds nothing; nothing read of what runs waits for none.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
invokes []string
|
||||
served []string
|
||||
nilRead bool
|
||||
holds string
|
||||
}{
|
||||
{"a verb no running build serves", []string{"store.store_put", "seat:mesh-catalog.modules"}, []string{"store.store_put"}, false, "seat:mesh-catalog.modules"},
|
||||
{"once a build serving it runs", []string{"seat:mesh-catalog.modules"}, []string{"seat:mesh-catalog.modules"}, false, ""},
|
||||
{"an edge left out on a cycle", nil, nil, false, ""},
|
||||
{"nothing read of what runs", []string{"seat:mesh-catalog.modules"}, nil, true, ""},
|
||||
// ADR 0614 §5: a module asking bus state is held while no build offering bus-state runs, and moves once one does.
|
||||
{"bus state no running build offers", nil, nil, false, graph.BusState},
|
||||
{"bus state a running build offers", nil, []string{graph.BusState}, false, ""},
|
||||
{"bus state its own module offers", nil, nil, false, ""},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Invokes = c.invokes
|
||||
if !c.nilRead {
|
||||
m.Served = func(v string) bool { return slices.Contains(c.served, v) }
|
||||
}
|
||||
if strings.HasPrefix(c.name, "bus state") {
|
||||
m.Manifest.BusState = []catalogue.BusStateEntry{{Name: "cache", Storage: "key-value"}}
|
||||
m.Offering = func(p string) bool { return slices.Contains(c.served, p) }
|
||||
if c.name == "bus state its own module offers" {
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
}
|
||||
words := graph.VerbWaitWords(c.holds, "mail", "g14")
|
||||
if c.holds == graph.BusState {
|
||||
words = graph.BusStateWaitWords("mail", "g14")
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if c.holds == "" {
|
||||
if out.Wants["g14"].FP != fNew {
|
||||
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out.Wants["g14"].FP == fNew || out.Rules["g14"].Rule != delivery.RuleProvisions || out.Rules["g14"].Words != words {
|
||||
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
// ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision
|
||||
// asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided
|
||||
// oidc moves the want, an asked one holds it and names oidc and never database.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
state, older string
|
||||
moves bool
|
||||
}{{"the replaced edge provided, the older edge retiring", provisions.Provided, provisions.Retiring, true},
|
||||
{"the replaced edge provided, the older edge asked", provisions.Provided, provisions.Asked, true},
|
||||
{"the replaced edge asked, the older edge retiring", provisions.Asked, provisions.Retiring, false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{"oidc"}
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
|
||||
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
|
||||
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
|
||||
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
|
||||
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
|
||||
{
|
||||
handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}}
|
||||
keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}}
|
||||
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh,
|
||||
AnsweredBy: catalogue.AnsweredByHelper}}}
|
||||
store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}}
|
||||
goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}}
|
||||
gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}}
|
||||
answering := func(p string) []catalogue.Manifest {
|
||||
for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} {
|
||||
if m.Provides[0].Name == p {
|
||||
return []catalogue.Manifest{m}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
manifest catalogue.Manifest
|
||||
missing string // the provision named, "" for none: the want moves
|
||||
}{
|
||||
{"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"database"}}, "database"},
|
||||
{"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""},
|
||||
{"a module not handed waits for its route, which the helper answers (grafana's route)",
|
||||
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"},
|
||||
{"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)",
|
||||
catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""},
|
||||
{"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
|
||||
Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Manifest, m.Answering = c.manifest, answering
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if c.missing == "" {
|
||||
if out.Wants["g14"].FP != fNew || len(m.MissingProvisions("g14")) != 0 {
|
||||
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out.Rules["g14"].Rule != delivery.RuleProvisions || !strings.Contains(out.Rules["g14"].Words,
|
||||
"provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 {
|
||||
t.Errorf("%s: %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
}
|
||||
// ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own
|
||||
// name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not.
|
||||
for _, c := range []struct {
|
||||
name, requires string
|
||||
offered bool
|
||||
}{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true},
|
||||
{"a provision nobody offers", "database", false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{c.requires}
|
||||
m.Answering = func(string) []catalogue.Manifest { return nil }
|
||||
m.Offered = func(p string) bool { return c.offered && p == c.requires }
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != delivery.RuleProvisions ||
|
||||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
|
||||
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
|
||||
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
|
||||
// healthy held photos, and novox offline left a module with no canary.
|
||||
for _, c := range []struct {
|
||||
name, module, provision, provider string
|
||||
have *delivery.Have // what novox runs of the module; nil: a new assignment
|
||||
before map[string]delivery.Want
|
||||
online bool
|
||||
want string
|
||||
}{
|
||||
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
|
||||
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
|
||||
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
|
||||
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
|
||||
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
|
||||
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
|
||||
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
|
||||
} {
|
||||
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
|
||||
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
|
||||
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
|
||||
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
|
||||
if c.have != nil {
|
||||
n.Haves["novox"][c.module] = *c.have
|
||||
}
|
||||
out := Wants(m, n, c.before, nil)
|
||||
w, written := out.Wants["novox"]
|
||||
if !written {
|
||||
w = c.before["novox"]
|
||||
}
|
||||
if w.FP != c.want || w.Removed {
|
||||
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
|
||||
}
|
||||
if _, on := out.Neither["novox"]; on {
|
||||
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
|
||||
}
|
||||
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
|
||||
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
|
||||
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
|
||||
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
|
||||
// stays and is said; a placed node's want is Self with no composition.
|
||||
selfAt := func(fp string) map[string]delivery.Want {
|
||||
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
|
||||
}
|
||||
atF := func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
set func(*Module, *Nodes)
|
||||
before string
|
||||
want delivery.Want // the want of g14 written; zero: none written
|
||||
rule int // the rule g14's holding names
|
||||
asked bool
|
||||
}{
|
||||
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
|
||||
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
|
||||
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
|
||||
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
|
||||
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
|
||||
rule: delivery.RuleNotHeld},
|
||||
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Held = true
|
||||
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
|
||||
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
|
||||
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
recording(m)
|
||||
m.Held = true
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
|
||||
}
|
||||
}, rule: delivery.RuleNotHeld, asked: true},
|
||||
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, rule: delivery.RuleComposed},
|
||||
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Self = func(node string) bool { return node == "g14" }
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if node == "g14" {
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, want: delivery.Want{FP: fNew, Self: true}},
|
||||
} {
|
||||
m, n := ready()
|
||||
c.set(&m, &n)
|
||||
before := selfAt(c.before)
|
||||
if c.name == "placed" {
|
||||
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
}
|
||||
out := Wants(m, n, before, nil)
|
||||
if got := out.Wants["g14"]; got != c.want {
|
||||
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
|
||||
}
|
||||
if got := out.Rules["g14"].Rule; got != c.rule {
|
||||
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
|
||||
}
|
||||
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
|
||||
t.Errorf("%s: record asks %+v", c.name, out.Record)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok {
|
||||
if _, err := delivery.EncodeWant(w); err != nil {
|
||||
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
@@ -449,7 +109,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
@@ -490,72 +150,6 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
||||
}
|
||||
|
||||
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
|
||||
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
|
||||
if out.Held != nil || len(out.Wants) != 0 {
|
||||
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
|
||||
}
|
||||
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
|
||||
len(out.NodeFailed) != 1 {
|
||||
t.Errorf("node failed = %+v", out.NodeFailed)
|
||||
}
|
||||
|
||||
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
|
||||
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
|
||||
three := m
|
||||
three.Nodes = []string{"ace", "g14", "novox"}
|
||||
failedHave := n.Haves["ace"]["mail"]
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
|
||||
n.Online["novox"] = true
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
|
||||
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if _, alone := out.NodeFailed["g14"]; !alone {
|
||||
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
|
||||
}
|
||||
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
|
||||
}
|
||||
if _, written := out.Wants["g14"]; written {
|
||||
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
|
||||
}
|
||||
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
delete(before, "novox")
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
|
||||
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
|
||||
}
|
||||
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
|
||||
|
||||
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
|
||||
// not one node's failure (D3). Either way F is held and ace is set back.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
online map[string]bool
|
||||
novox delivery.Health
|
||||
}{
|
||||
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
|
||||
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
|
||||
} {
|
||||
n.Online = c.online
|
||||
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
|
||||
if c.novox == delivery.HealthOK {
|
||||
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
||||
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
|
||||
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld {
|
||||
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
||||
@@ -776,13 +370,11 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
||||
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
||||
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
|
||||
// backs its streams up before it applies.
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
|
||||
t.Errorf("the bus did not move by the rules: %+v", out)
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
@@ -937,13 +529,6 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
|
||||
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
|
||||
m.Self = func(string) bool { return true }
|
||||
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer in a placed node's want was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
@@ -1003,16 +588,10 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
||||
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
||||
}
|
||||
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
|
||||
with(delivery.PolicyRecord, "person")
|
||||
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
||||
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
|
||||
// bus has no bar of its own (ADR 0601 §1).
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
||||
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
@@ -1021,8 +600,8 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
||||
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1210,12 +789,10 @@ func x11() (Module, Nodes) {
|
||||
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
out := Wants(m, n, nil, nil)
|
||||
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
|
||||
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
|
||||
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
|
||||
}
|
||||
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
|
||||
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||
}
|
||||
}
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("left on neither list: %+v", out.Neither)
|
||||
@@ -1253,22 +830,11 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if what == "an old-form build" {
|
||||
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
|
||||
}
|
||||
continue
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("%s: shanks's want = %+v, want F", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("%s: shanks on neither list = %+v", what, h)
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||
}
|
||||
}
|
||||
m, n := x11()
|
||||
@@ -1277,12 +843,10 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||
}
|
||||
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
|
||||
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
|
||||
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
|
||||
}
|
||||
if w := out.Wants["g14"]; w.FP != fNew {
|
||||
t.Errorf("the canary g14's want = %+v, want F", w)
|
||||
for node, w := range out.Wants {
|
||||
if w.FP != fOld {
|
||||
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1293,12 +857,11 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
m, n := x11()
|
||||
delete(n.Haves["shanks"], "screen-lock")
|
||||
out := Wants(m, n, nil, nil)
|
||||
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
|
||||
if w := out.Wants["shanks"]; w.FP != fNew {
|
||||
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
|
||||
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||
}
|
||||
m.Built = false
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
@@ -1340,7 +903,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
oldForm := func(n Nodes, node string) {
|
||||
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
@@ -1374,16 +937,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||
}
|
||||
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
|
||||
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||
m, n = x11()
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
|
||||
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||
}
|
||||
if _, on := out.Neither["shanks"]; on {
|
||||
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
|
||||
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||
}
|
||||
// F not granted: nothing, even with the canary healthy
|
||||
m, n = x11()
|
||||
|
||||
@@ -1,332 +0,0 @@
|
||||
// Package judge is how the seat mesh-placement's holder judges a proposal (novox/hq ADR 0567 §3, §4, as ADR 0593 §8
|
||||
// changes them): what it changes on each node by its placement (Intended), every node answering it, and the verdict of
|
||||
// their answers (JudgeAnswers). The assigner's own (ADR 0593 §1), moved back from mesh-sdk's compose/judge: the holder
|
||||
// composes nothing, since every composition is its node's engine's, and a node answers through the candidates path
|
||||
// with what it composed. It holds no bus client and no timer: the holder gives it what it read.
|
||||
package judge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// AnswerBound is how long the holder waits for a node's answer after its candidate was written: the time to live of the
|
||||
// node's pulse key (delivery.PulseAnswerKey), whose expiry names it as giving no answer, never passed. The same answer
|
||||
// bound a check gives a node (ADR 0593 §8, §7: 60 seconds, nec.AnswerBound). A variable so a test on the bus
|
||||
// can shorten it.
|
||||
var AnswerBound = nec.AnswerBound
|
||||
|
||||
// Intended is, by node, the modules whose placed entry the proposal changes there, and the nodes that answer it: every
|
||||
// node (ADR 0593 §8, changing ADR 0567 §4's list of the nodes that must answer), sorted. current is the placement as it
|
||||
// stands, laid the proposal's placements by node (nil removes the node's key) and nodes every node with a record. A
|
||||
// node's set holds the modules the proposal adds, removes or whose placed entry differs on that node; a node with no
|
||||
// key now gets an empty set (§7: the seed is answered as the same). What a node's compositions change beyond its own
|
||||
// placement it says itself in its answer, marked in the brief as not computed.
|
||||
func Intended(current map[string]delivery.Placement, laid map[string]*delivery.Placement, nodes []string) (intended map[string]map[string]bool, must []string) {
|
||||
intended = map[string]map[string]bool{}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n] = true
|
||||
}
|
||||
for n := range laid {
|
||||
all[n] = true
|
||||
}
|
||||
for node := range all {
|
||||
must = append(must, node)
|
||||
set := map[string]bool{}
|
||||
intended[node] = set
|
||||
before, placed := current[node]
|
||||
after, changesKey := laid[node]
|
||||
if !changesKey || !placed {
|
||||
continue // untouched, or new to placement: every answer must be the same
|
||||
}
|
||||
var afterModules []delivery.PlacedModule
|
||||
if after != nil {
|
||||
afterModules = after.Modules
|
||||
}
|
||||
entries := map[string]delivery.PlacedModule{}
|
||||
for _, m := range before.Modules {
|
||||
entries[m.Module] = m
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range afterModules {
|
||||
seen[m.Module] = true
|
||||
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
|
||||
set[m.Module] = true
|
||||
}
|
||||
}
|
||||
for name := range entries {
|
||||
if !seen[name] {
|
||||
set[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(must)
|
||||
return intended, must
|
||||
}
|
||||
|
||||
// JudgeAnswers is the verdict of a proposal's answers (§4). answers is by node its HaveNode.Candidates entry for the
|
||||
// proposal (nil when it has none) and revisions by node the revision of its candidate key: an answer counts when it
|
||||
// names the proposal and that revision. It fails on a refusal by a node that must answer; a node of Must with no answer
|
||||
// that counts is waited for while its pulse key stands, and once it has ended (pulsing false: the bound's expiry, heard
|
||||
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
|
||||
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
|
||||
// shown in the ask's brief. The seed alone fails on any difference.
|
||||
func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
|
||||
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) delivery.PlacementVerdict {
|
||||
v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
|
||||
Answers: map[string]delivery.CandidateAnswer{}}
|
||||
for node, set := range intended {
|
||||
for m := range set {
|
||||
v.Intended[node] = append(v.Intended[node], m)
|
||||
}
|
||||
sort.Strings(v.Intended[node])
|
||||
}
|
||||
must := p.Must
|
||||
if len(must) == 0 {
|
||||
must = p.Nodes()
|
||||
}
|
||||
for _, node := range must {
|
||||
a := answers[node]
|
||||
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
|
||||
v.Silent = append(v.Silent, node)
|
||||
continue
|
||||
}
|
||||
v.Answers[node] = *a
|
||||
if a.Refused != "" {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s cannot compose the proposal: %s", node, a.Refused))
|
||||
continue
|
||||
}
|
||||
for _, r := range a.Refusals {
|
||||
v.Fails = append(v.Fails, fmt.Sprintf("%s refuses the proposal for %s: %s", node, r.Module, r.Why))
|
||||
}
|
||||
want := intended[node]
|
||||
differs := func(module, words string) {
|
||||
if p.Seed {
|
||||
// The seed carries what its nodes found, each named in the brief for the operator who approves it.
|
||||
v.Carried[node] = append(v.Carried[node], words)
|
||||
return
|
||||
}
|
||||
if !want[module] {
|
||||
v.Unintended[node] = append(v.Unintended[node], module)
|
||||
}
|
||||
}
|
||||
for _, d := range a.Differs {
|
||||
differs(d.Module, differenceWords(d))
|
||||
}
|
||||
for _, d := range a.Added {
|
||||
differs(d.Module, "would run "+d.Module)
|
||||
}
|
||||
for _, m := range a.Removed {
|
||||
differs(m, "would stop running "+m)
|
||||
}
|
||||
sort.Strings(v.Unintended[node])
|
||||
}
|
||||
if len(v.Intended) == 0 {
|
||||
v.Intended = nil
|
||||
}
|
||||
if len(v.Unintended) == 0 {
|
||||
v.Unintended = nil
|
||||
}
|
||||
if len(v.Carried) == 0 {
|
||||
v.Carried = nil
|
||||
}
|
||||
if len(v.Answers) == 0 {
|
||||
v.Answers = nil
|
||||
}
|
||||
var waiting, gone []string
|
||||
for _, node := range v.Silent {
|
||||
named := node
|
||||
if online != nil && !online[node] {
|
||||
named += " (offline)"
|
||||
}
|
||||
if pulsing[node] {
|
||||
waiting = append(waiting, named)
|
||||
} else {
|
||||
gone = append(gone, named)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("waits for the answers of %s, each for %s from its candidate's write",
|
||||
strings.Join(waiting, ", "), AnswerBound))
|
||||
}
|
||||
if len(gone) > 0 {
|
||||
// ADR 0593 §8: decided on the answers it has; that node's own composition judges again when it applies (§9).
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("no answer from %s within %s of its candidate's write: decided on the answers "+
|
||||
"it has, and that node's own composition judges again when it applies", strings.Join(gone, ", "), AnswerBound))
|
||||
}
|
||||
switch {
|
||||
case len(v.Fails) > 0:
|
||||
v.State = delivery.VerdictFail
|
||||
case len(waiting) > 0:
|
||||
v.State = delivery.VerdictWaits
|
||||
default:
|
||||
v.State = delivery.VerdictPass
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Brief is the verdict as the ask's brief shows it, by node (§5): what the proposal changes and what each node answered
|
||||
// beside it, marked, and what stays held as the node holds it today, with the proposal's why.
|
||||
func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s (proposed by %s)", p.Why, p.By)
|
||||
nodes := map[string]bool{}
|
||||
for n := range v.Intended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n := range v.Unintended {
|
||||
nodes[n] = true
|
||||
}
|
||||
for _, n := range p.Nodes() {
|
||||
nodes[n] = true
|
||||
}
|
||||
for n, a := range v.Answers {
|
||||
if len(a.Held) > 0 {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
for n := range v.Carried {
|
||||
nodes[n] = true
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for n := range nodes {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, "; %s:", n)
|
||||
if m := v.Intended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, " changes %s", strings.Join(m, ", "))
|
||||
} else {
|
||||
b.WriteString(" composes the same")
|
||||
}
|
||||
if m := v.Unintended[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; also differs, not computed: %s", strings.Join(m, ", "))
|
||||
}
|
||||
if m := v.Carried[n]; len(m) > 0 {
|
||||
fmt.Fprintf(&b, "; the seed carries: %s", strings.Join(m, "; "))
|
||||
}
|
||||
if held := v.Answers[n].Held; len(held) > 0 {
|
||||
var names []string
|
||||
for _, h := range held {
|
||||
today := h.Today
|
||||
if today == "" {
|
||||
today = "held"
|
||||
}
|
||||
names = append(names, h.Module+" ("+today+")")
|
||||
}
|
||||
fmt.Fprintf(&b, "; stays as today: %s", strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
|
||||
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
|
||||
// field), and how many fields more; or the resource and why when no body is shown.
|
||||
func differenceWords(d delivery.ModuleDiffers) string {
|
||||
for _, b := range d.Bodies {
|
||||
if len(b.Paths) == 0 {
|
||||
continue
|
||||
}
|
||||
path := b.Paths[0]
|
||||
words := fmt.Sprintf("%s: %s %s → %s", d.Module, fieldName(b.Running, b.Composed, path),
|
||||
valueAt(b.Running, path), valueAt(b.Composed, path))
|
||||
if n := len(b.Paths) - 1; n > 0 {
|
||||
words += fmt.Sprintf(" (and %d more fields)", n)
|
||||
}
|
||||
return words
|
||||
}
|
||||
if d.Resource != "" {
|
||||
return fmt.Sprintf("%s: %s (%s)", d.Module, d.Resource, d.Why)
|
||||
}
|
||||
return fmt.Sprintf("%s: %s", d.Module, d.Why)
|
||||
}
|
||||
|
||||
// shownValue bounds a value named in a brief.
|
||||
const shownValue = 60
|
||||
|
||||
// walk follows a differing path (delivery.DifferingPaths: keys joined by dots, a key may hold dots, and a JSON value
|
||||
// written in a string is read as JSON) through a body, answering the keys it passed and the value at its end.
|
||||
func walk(body, path string) (keys []string, value any, ok bool) {
|
||||
var v any
|
||||
if json.Unmarshal([]byte(body), &v) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest := path
|
||||
for rest != "" {
|
||||
if s, isString := v.(string); isString {
|
||||
var inner any
|
||||
if json.Unmarshal([]byte(s), &inner) != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
v = inner
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
best := ""
|
||||
for k := range t {
|
||||
if (rest == k || strings.HasPrefix(rest, k+".")) && len(k) > len(best) {
|
||||
best = k
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, best), t[best], strings.TrimPrefix(strings.TrimPrefix(rest, best), ".")
|
||||
case []any:
|
||||
seg, after, _ := strings.Cut(rest, ".")
|
||||
n, err := strconv.Atoi(seg)
|
||||
if err != nil || n < 0 || n >= len(t) {
|
||||
return nil, nil, false
|
||||
}
|
||||
keys, v, rest = append(keys, seg), t[n], after
|
||||
default:
|
||||
return nil, nil, false
|
||||
}
|
||||
}
|
||||
return keys, v, true
|
||||
}
|
||||
|
||||
// valueAt is the value at path in a shown body, cut to shownValue, or why it is not shown.
|
||||
func valueAt(body, path string) string {
|
||||
if body == "" {
|
||||
return "(none)"
|
||||
}
|
||||
_, v, ok := walk(body, path)
|
||||
if !ok {
|
||||
return "(value cut)"
|
||||
}
|
||||
s, isString := v.(string)
|
||||
if !isString {
|
||||
raw, _ := json.Marshal(v)
|
||||
s = string(raw)
|
||||
}
|
||||
if r := []rune(s); len(r) > shownValue {
|
||||
s = string(r[:shownValue-1]) + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fieldName is the differing field as a person reads it: its parent's name and its own, `PARENT[key]`, from whichever
|
||||
// body reaches it; the path's end when neither does.
|
||||
func fieldName(running, composed, path string) string {
|
||||
for _, body := range []string{running, composed} {
|
||||
if keys, _, ok := walk(body, path); ok && len(keys) > 0 {
|
||||
if len(keys) == 1 {
|
||||
return keys[0]
|
||||
}
|
||||
return keys[len(keys)-2] + "[" + keys[len(keys)-1] + "]"
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package judge
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0567 §3, §4 (rows moved from the assigner's check of a pull request's placement, ADR 0559 §5): what a proposal
|
||||
// changes by node, and the verdict of the answers: differences shown and failing nothing, a refusal and a silent node
|
||||
// failing, a later answer judging again, the seed failing on any difference.
|
||||
func TestJudge(t *testing.T) {
|
||||
current := map[string]delivery.Placement{"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "server": {Node: "server"}}
|
||||
laid := map[string]*delivery.Placement{
|
||||
"anchor": {Node: "anchor", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}},
|
||||
"laptop": {Node: "laptop", Modules: []delivery.PlacedModule{{Module: "web"}}}}
|
||||
// ADR 0593 §8: every node answers; what the placement changes is read from the placement alone.
|
||||
got, must := Intended(current, laid, []string{"anchor", "laptop", "server"})
|
||||
if !got["anchor"]["mail"] || !got["anchor"]["web"] || len(got["laptop"]) != 0 || len(got["server"]) != 0 ||
|
||||
!slices.Equal(must, []string{"anchor", "laptop", "server"}) {
|
||||
t.Errorf("intended = %v, must %v: want anchor's mail and web, nothing elsewhere, every node answering", got, must)
|
||||
}
|
||||
|
||||
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
|
||||
p := delivery.PlacementProposal{ID: "p-one", By: "op", Why: "move web", Written: at, Base: map[string]uint64{"anchor": 3, "laptop": 0},
|
||||
Must: []string{"anchor", "laptop"}}
|
||||
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
|
||||
intended := map[string]map[string]bool{"anchor": {"web": true}}
|
||||
same := func(rev uint64) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
|
||||
}
|
||||
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
|
||||
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
seed bool
|
||||
answers map[string]*delivery.CandidateAnswer
|
||||
pulsing map[string]bool
|
||||
state string
|
||||
words string
|
||||
marked string
|
||||
}{
|
||||
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
|
||||
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
|
||||
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
|
||||
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
|
||||
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
|
||||
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, delivery.VerdictWaits, "laptop (offline)", ""},
|
||||
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
|
||||
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
|
||||
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
|
||||
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
|
||||
} {
|
||||
q := p
|
||||
q.Seed = row.seed
|
||||
v := JudgeAnswers(q, intended, row.answers, revisions, map[string]bool{"anchor": true}, row.pulsing, at)
|
||||
all := strings.Join(append(append([]string{}, v.Fails...), v.Notes...), "; ")
|
||||
if v.State != row.state || !strings.Contains(all, row.words) {
|
||||
t.Errorf("%s: %s, %q", row.name, v.State, all)
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass && !strings.Contains(Brief(q, v), "the seed carries: node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → (value cut) (and 1 more fields)") {
|
||||
t.Errorf("%s: the brief does not name it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.seed && row.state == delivery.VerdictPass {
|
||||
whole := row.answers["laptop"].Differs[0]
|
||||
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
|
||||
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
|
||||
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
|
||||
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
|
||||
t.Errorf("the difference is named %q", got)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(row.name, "a module held as today") && !strings.Contains(Brief(q, v), "stays as today: spiegel (held)") {
|
||||
t.Errorf("%s: the brief does not show it: %s", row.name, Brief(q, v))
|
||||
}
|
||||
if row.marked != "" && (!slices.Contains(v.Unintended["anchor"], row.marked) || !strings.Contains(Brief(q, v), "not computed: "+row.marked)) {
|
||||
t.Errorf("%s: the difference is not marked: %+v, %s", row.name, v.Unintended, Brief(q, v))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
func load(t *testing.T) catalogue.Manifest {
|
||||
@@ -18,9 +15,9 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
|
||||
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -35,13 +32,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
|
||||
// every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1).
|
||||
// DLV-F2r (ADR 0615 §1, ADR 0631 §1): the indexer's manifests, modules and module graph (whose verb edges a want waits
|
||||
// on, ADR 0614 §2, §3) are read through its first views and views family, none of the indexer's states by reads.
|
||||
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source",
|
||||
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
|
||||
"mesh-provisioner.provisions", "node-engine.have"}
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -49,9 +42,7 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
|
||||
"engine-reads": 1}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -60,64 +51,18 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1906/D3: it says its conditions on the operator channel; it asks the Go build seat nothing since the composition is
|
||||
// the node-engine's (ADR 0593 §7).
|
||||
if !slices.Equal(m.Uses, []string{"operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly operator-channel", m.Uses)
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its five verbs (adopt and hand-back went with the
|
||||
// controller, ADR 0590); keeps the seat's events on its stream placement for 30 days; calls the hand-act log; and reads
|
||||
// mesh-config's resolved values to compose every node a proposal touches.
|
||||
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
|
||||
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again"}) ||
|
||||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its five verbs and mesh-wants with its two", m.Claims)
|
||||
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||
// calls the controller's warranted and the bus node's backup holder.
|
||||
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
|
||||
}
|
||||
// ADR 0621 §1: the first-view tools placement, seat-holders and wants and design 25 § 12.2's six read tools, of the
|
||||
// program's bundle, which runs in a runtime of its own on the assigner's account; ADR 0618 §3: and assigner_engine_reads,
|
||||
// which lists and asks again what each node's engine reads.
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "placement", "seat-holders", "wants",
|
||||
"assigner_read_placement", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_asks", "assigner_read_differences",
|
||||
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools and assigner_engine_reads", m.Tools)
|
||||
}
|
||||
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
|
||||
t.Errorf("runs-as = %q: the bundle runs on the assigner's own account (ADR 0621 §1)", m.RunsAs)
|
||||
}
|
||||
// ADR 0621 §2: each first-view tool answers exactly its readers, never the assigner.
|
||||
callers := map[string]string{"placement": "mesh-grants,node-tools,mesh-delivery-indexer", "seat-holders": "node-tools", "wants": "mesh-grants"}
|
||||
env := m.Build.Artifacts[0].Env
|
||||
for tool, list := range callers {
|
||||
if env[views.CallersEnv(tool)] != list || views.ParseCallers(env[views.CallersEnv(tool)]).Allows("n1."+m.Module) {
|
||||
t.Errorf("%s callers = %q, want %q", tool, env[views.CallersEnv(tool)], list)
|
||||
}
|
||||
}
|
||||
// ADR 0619 §2, ADR 0615 §5: its views family beside placement on its old streams section.
|
||||
if len(m.Streams) != 2 || m.Streams[1].Name != "assigner-views" || !slices.Equal(m.Streams[1].Events, []string{views.EventPattern}) ||
|
||||
m.Streams[1].MaxAgeDays != views.StreamDays || m.Streams[1].MaxMiB != views.StreamMiB || m.Streams[1].Retention != "" {
|
||||
t.Errorf("streams = %+v, want placement and assigner-views", m.Streams)
|
||||
}
|
||||
if m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
// ADR 0618 §4: and it hears each node's engine say its reads changed.
|
||||
if !slices.Equal(m.Consumes, []string{"mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "node-engine.manifest-reads-changed"}) {
|
||||
t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules and the engines' reads changed", m.Consumes)
|
||||
}
|
||||
if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now",
|
||||
"seat:node-engine.manifest-reads"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0618 §3
|
||||
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
|
||||
@@ -2,58 +2,28 @@
|
||||
"module": "mesh-delivery-assigner",
|
||||
"version": "1",
|
||||
"slug": "assigner",
|
||||
"runs-as": "mesh-delivery-assigner",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-placement",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"placement",
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "mesh-wants",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"upgrade",
|
||||
"bus"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-grants.grants",
|
||||
"mesh-host.composed",
|
||||
"mesh-provisioner.provisions",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-indexer.views.graph",
|
||||
"mesh-delivery-indexer.views.manifests",
|
||||
"mesh-delivery-indexer.views.modules",
|
||||
"node-engine.manifest-reads-changed"
|
||||
],
|
||||
"invokes": [
|
||||
"mesh-delivery-indexer.graph",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.now",
|
||||
"seat:node-engine.manifest-reads"
|
||||
"seat:node-backup.backed-up"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
@@ -61,48 +31,14 @@
|
||||
"tools": [
|
||||
"assigner_put",
|
||||
"assigner_release",
|
||||
"assigner_puts",
|
||||
"placement",
|
||||
"seat-holders",
|
||||
"wants",
|
||||
"assigner_read_placement",
|
||||
"assigner_read_seat_holders",
|
||||
"assigner_read_wants",
|
||||
"assigner_read_findings",
|
||||
"assigner_read_asks",
|
||||
"assigner_read_differences",
|
||||
"assigner_first_views",
|
||||
"assigner_first_view_again",
|
||||
"assigner_engine_reads"
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "placement", "history": 64 },
|
||||
{ "name": "proposals", "history": 5 },
|
||||
{ "name": "candidates", "history": 5 },
|
||||
{ "name": "pulses", "key-ttl": true },
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 },
|
||||
{ "name": "findings", "history": 5 },
|
||||
{ "name": "policies", "history": 20 },
|
||||
{ "name": "seat-holders", "history": 5 },
|
||||
{ "name": "view-marks", "history": 1 },
|
||||
{ "name": "first-views", "history": 1 },
|
||||
{ "name": "engine-reads", "history": 1 }
|
||||
],
|
||||
"streams": [
|
||||
{
|
||||
"name": "placement",
|
||||
"events": [
|
||||
"placement-proposed",
|
||||
"placement-judged",
|
||||
"placement-changed",
|
||||
"placement-withdrawn"
|
||||
],
|
||||
"max-age-days": 30
|
||||
},
|
||||
{ "name": "assigner-views", "events": ["views.**"], "max-age-days": 7, "max-mib": 256 }
|
||||
{ "name": "wants", "history": 20 }
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
@@ -152,12 +88,7 @@
|
||||
"binary": "mesh-delivery-assigner",
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
],
|
||||
"env": {
|
||||
"MESH_VIEW_CALLERS_PLACEMENT": "mesh-grants,node-tools,mesh-delivery-indexer",
|
||||
"MESH_VIEW_CALLERS_SEAT_HOLDERS": "node-tools",
|
||||
"MESH_VIEW_CALLERS_WANTS": "mesh-grants"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,316 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
|
||||
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
|
||||
// and the asker reads the first such key that is not running. The builder and the checker share it.
|
||||
|
||||
// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
|
||||
type AskPublisher interface {
|
||||
PublishAsk(ctx context.Context, subject, id string, data []byte) error
|
||||
}
|
||||
|
||||
// AskWaiter watches an ask's steps: every value a key `*.<ask id>` of the build module's steps state takes, running or
|
||||
// ended, on the channel it answers, until ctx ends. A watch, not a poll.
|
||||
type AskWaiter interface {
|
||||
WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error)
|
||||
}
|
||||
|
||||
// Asker puts asks and waits for their steps.
|
||||
type Asker struct {
|
||||
Publisher AskPublisher
|
||||
Waiter AskWaiter
|
||||
// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
|
||||
// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
|
||||
// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Put puts a on its stream: the seat's accept subject of its language and kind.
|
||||
func (k Asker) Put(ctx context.Context, a delivery.StepAsk) error {
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.Publisher.PublishAsk(ctx, subject, a.ID, raw); err != nil {
|
||||
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
|
||||
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
|
||||
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
|
||||
func (k Asker) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
watch, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
var bound, outerCap <-chan time.Time
|
||||
var timer, capTimer *time.Timer
|
||||
defer func() {
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
}()
|
||||
running, since, phase, phaseAt := "", time.Time{}, "", time.Time{}
|
||||
standStill := func() error {
|
||||
if phase == "" {
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound)
|
||||
}
|
||||
return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s",
|
||||
a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.Step{}, ctx.Err()
|
||||
case <-bound:
|
||||
return delivery.Step{}, standStill()
|
||||
case <-outerCap:
|
||||
return delivery.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended",
|
||||
a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
|
||||
case step, ok := <-steps:
|
||||
if !ok {
|
||||
return delivery.Step{}, errors.New("the watch of the ask's step ended")
|
||||
}
|
||||
if seen := stepSeenOf(ctx); seen != nil {
|
||||
seen(step)
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return step, nil
|
||||
}
|
||||
if k.Bound <= 0 {
|
||||
continue
|
||||
}
|
||||
newRun := step.Node != running || !step.Started.Equal(since)
|
||||
if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) {
|
||||
continue
|
||||
}
|
||||
// A node took the ask, or took it again after its run stopped (another node, or the same node's build module
|
||||
// replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the
|
||||
// step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to
|
||||
// three bounds from its start (issue 1930).
|
||||
running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt
|
||||
last := step.Started
|
||||
if step.PhaseAt.After(last) {
|
||||
last = step.PhaseAt
|
||||
}
|
||||
left := k.Bound
|
||||
if !last.IsZero() {
|
||||
left = time.Until(last.Add(k.Bound))
|
||||
}
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
timer = time.NewTimer(max(left, 0))
|
||||
bound = timer.C
|
||||
if newRun {
|
||||
if capTimer != nil {
|
||||
capTimer.Stop()
|
||||
}
|
||||
outer := 3 * k.Bound
|
||||
if !step.Started.IsZero() {
|
||||
outer = time.Until(step.Started.Add(3 * k.Bound))
|
||||
}
|
||||
capTimer = time.NewTimer(max(outer, 0))
|
||||
outerCap = capTimer.C
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type stepSeenKey struct{}
|
||||
|
||||
// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node
|
||||
// that took it, each phase it says while running (delivery.Step.Phase), and its end. A try names its asks' progress so
|
||||
// (issue 1890). seen runs in the waiting goroutine and must not block.
|
||||
func WithStepSeen(ctx context.Context, seen func(delivery.Step)) context.Context {
|
||||
return context.WithValue(ctx, stepSeenKey{}, seen)
|
||||
}
|
||||
|
||||
func stepSeenOf(ctx context.Context) func(delivery.Step) {
|
||||
seen, _ := ctx.Value(stepSeenKey{}).(func(delivery.Step))
|
||||
return seen
|
||||
}
|
||||
|
||||
// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
|
||||
func (k Asker) Ask(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
return k.Passed(k.Wait(ctx, a))
|
||||
}
|
||||
|
||||
// Passed is step when it passed, else an error naming where it ended and why.
|
||||
func (k Asker) Passed(step delivery.Step, err error) (delivery.Step, error) {
|
||||
if err != nil {
|
||||
return step, err
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
|
||||
}
|
||||
return step, nil
|
||||
}
|
||||
|
||||
// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
|
||||
// of `*.<ask id>` in the build module's steps bucket.
|
||||
type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
||||
msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
|
||||
msg.Header.Set(jetstream.MsgIDHeader, id)
|
||||
_, err := s.JS.PublishMsg(ctx, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
|
||||
// once, and every change after.
|
||||
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
w, err := kv.Watch(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
|
||||
}
|
||||
select {
|
||||
case out <- step:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
|
||||
// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for
|
||||
// concurrent use.
|
||||
type FakeAsks struct {
|
||||
Answer func(a delivery.StepAsk) delivery.Step
|
||||
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
steps map[string][]delivery.Step
|
||||
watchers map[string][]chan delivery.Step
|
||||
}
|
||||
|
||||
// PublishAsk is AskPublisher's.
|
||||
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
|
||||
a, err := delivery.DecodeStepAsk(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.asks = append(f.asks, a)
|
||||
f.mu.Unlock()
|
||||
if f.Answer != nil {
|
||||
f.Write(f.Answer(a))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write writes an ask's step, as a build node does.
|
||||
func (f *FakeAsks) Write(step delivery.Step) {
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
f.steps[step.ID] = append(f.steps[step.ID], step)
|
||||
ws := append([]chan delivery.Step(nil), f.watchers[step.ID]...)
|
||||
f.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- step
|
||||
}
|
||||
}
|
||||
|
||||
// End is Write: kept for the tests that end a step.
|
||||
func (f *FakeAsks) End(step delivery.Step) { f.Write(step) }
|
||||
|
||||
// WatchAsk is AskWaiter's: what was written so far, then every write.
|
||||
func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan delivery.Step, error) {
|
||||
out := make(chan delivery.Step, 64)
|
||||
f.mu.Lock()
|
||||
if f.steps == nil {
|
||||
f.steps, f.watchers = map[string][]delivery.Step{}, map[string][]chan delivery.Step{}
|
||||
}
|
||||
for _, s := range f.steps[askID] {
|
||||
out <- s
|
||||
}
|
||||
f.watchers[askID] = append(f.watchers[askID], out)
|
||||
f.mu.Unlock()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
f.mu.Lock()
|
||||
ws := f.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == out {
|
||||
f.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Asks are the asks put so far.
|
||||
func (f *FakeAsks) Asks() []delivery.StepAsk {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), f.asks...)
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// 1887/A4a, 1887/AC3 on the real bus: an ask put twice under one id is one message, and the asker reads the step a node
|
||||
// wrote under `<node>.<ask id>`.
|
||||
func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Subjects: []string{"mesh.seat.node-build-go.accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket("mesh-build-go")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus := JetStreamAsks{JS: js}
|
||||
k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
|
||||
a := goAsk("mvl0frv18d1960")
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := k.Put(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
s, _ := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
||||
if info, _ := s.Info(ctx); info.State.Msgs != 1 {
|
||||
t.Fatalf("%d messages for one ask put twice", info.State.Msgs)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
running, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
_, _ = kv.Put(ctx, key, running)
|
||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: a.ID, Tool: "delivery ask", Node: "ace", State: delivery.StepPassed, Started: time.Now(),
|
||||
Result: json.RawMessage(`{"artifacts":[]}`)})
|
||||
_, _ = kv.Put(ctx, key, passed)
|
||||
}()
|
||||
step, err := k.Passed(k.Wait(ctx, a))
|
||||
if err != nil || step.Node != "ace" {
|
||||
t.Fatalf("step %+v, %v", step, err)
|
||||
}
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func goAsk(id string) delivery.StepAsk {
|
||||
return delivery.StepAsk{ID: id, Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now(),
|
||||
Build: &delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}}
|
||||
}
|
||||
|
||||
// 1887/A4a: an asker puts the ask once and answers the step a build node wrote for it, naming the node; one that did not
|
||||
// pass is an error naming the node and why; a wait of an ask already put puts nothing.
|
||||
func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
f := &FakeAsks{Answer: func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "ace", State: delivery.StepPassed, Result: []byte(`{"artifacts":[]}`)}
|
||||
}}
|
||||
k := Asker{Publisher: f, Waiter: f}
|
||||
var seen []delivery.Step // issue 1890: a waiter that asks to see each value of the step sees it
|
||||
step, err := k.Ask(WithStepSeen(context.Background(), func(s delivery.Step) { seen = append(seen, s) }), goAsk("mvl0frv18d1957"))
|
||||
if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" {
|
||||
t.Fatalf("step %+v, %v, asks %d, seen %+v", step, err, len(f.Asks()), seen)
|
||||
}
|
||||
f.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
return delivery.Step{ID: a.ID, Node: "shanks", State: delivery.StepRefused, Reason: "go 1.27 installed"}
|
||||
}
|
||||
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
|
||||
t.Fatalf("a refused step: %v", err)
|
||||
}
|
||||
// review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step
|
||||
// running, the step is bounded from its start, and the error names the node.
|
||||
g := &FakeAsks{}
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
|
||||
waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded {
|
||||
t.Fatalf("an ask no node took: %v, want it still waiting when ctx ended", err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1961", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1961")); err == nil || !strings.Contains(err.Error(), "on ace did not end within") {
|
||||
t.Fatalf("a step that never ends: %v", err)
|
||||
}
|
||||
if len(g.Asks()) != 0 {
|
||||
t.Fatal("a wait put an ask")
|
||||
}
|
||||
// Issue 1930: the bound is on a step that stands still, not on its age. A step past one bound from its start that entered a
|
||||
// new phase inside the bound is waited for until it ends; one that stood a whole bound in a phase fails at once, naming
|
||||
// the phase; one three bounds from its start ends, said as kept moving. The steps' own times stand for the clock, so the
|
||||
// bound is an hour no loaded machine reaches and no case waits on a pause (the check of 2026-11-02 22:35 tripped one
|
||||
// of 60 ms).
|
||||
k = Asker{Publisher: g, Waiter: g, Bound: time.Hour}
|
||||
now := time.Now()
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepRunning, Started: now.Add(-90 * time.Minute), Phase: "pb", PhaseAt: now})
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "ace", State: delivery.StepPassed})
|
||||
if step, err := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || step.State != delivery.StepPassed {
|
||||
t.Fatalf("a step that kept moving past its bound: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1981", Node: "ace", State: delivery.StepRunning, Started: now.Add(-time.Hour), Phase: "integration: tests", PhaseAt: now.Add(-time.Hour)})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1981")); err == nil || !strings.Contains(err.Error(), `stood still in "integration: tests"`) {
|
||||
t.Fatalf("a step that stood still in a phase: %v", err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1982", Node: "ace", State: delivery.StepRunning, Started: now.Add(-3 * time.Hour), Phase: "qz", PhaseAt: now})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1982")); err == nil || !strings.Contains(err.Error(), "kept moving") {
|
||||
t.Fatalf("a step that never stops moving: %v", err)
|
||||
}
|
||||
// Issue 734: an ask the build seat gave up on (DEAD_LETTERS keeps its copy with the ask's id) ends its wait as failed
|
||||
// with the words, though a node had taken it; under NotTakenBound one no node takes ends at the bound; a step written
|
||||
// first is answered as the node wrote it.
|
||||
dead := &DeadLetters{}
|
||||
lw := LostAskWaiter{Inner: g, Dead: dead}
|
||||
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1970", Node: "ace", State: delivery.StepRunning, Started: time.Now()})
|
||||
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
|
||||
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
|
||||
if !ok {
|
||||
t.Fatal("a dead letter's headers were not read")
|
||||
}
|
||||
go dead.Heard(given)
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1970"))
|
||||
if err != nil || step.State != delivery.StepFailed || !IsLost(step) ||
|
||||
!strings.Contains(step.Reason, LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z") {
|
||||
t.Fatalf("a given-up ask: %+v, %v", step, err)
|
||||
}
|
||||
if _, err := k.Passed(step, nil); err == nil || !strings.Contains(err.Error(), "on "+LostNode+" failed: "+LostWords) {
|
||||
t.Fatalf("a given-up ask's error: %v", err)
|
||||
}
|
||||
step, err = k.Wait(NotTakenBound(context.Background(), 20*time.Millisecond), goAsk("mvl0frv18d1971"))
|
||||
if err != nil || !IsNotTaken(step) || IsLost(step) {
|
||||
t.Fatalf("an ask no node took: %+v, %v", step, err)
|
||||
}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1972", Node: "g14", State: delivery.StepPassed})
|
||||
if step, err = k.Wait(NotTakenBound(context.Background(), time.Minute), goAsk("mvl0frv18d1972")); err != nil || step.Node != "g14" {
|
||||
t.Fatalf("a step written: %+v, %v", step, err)
|
||||
}
|
||||
// Review round 1 of issue 734: a hearing the bus refuses at start is deaf and says so, never holds the start up, and
|
||||
// tries again on the next ask waited for; heard, it says so and hands the given-up asks on.
|
||||
fol := &refusingFollower{refuse: 1}
|
||||
var mu sync.Mutex
|
||||
var told []Hearing
|
||||
hearing := NewDeadLetterHearing(context.Background(), fol, []string{delivery.AskGo}, dead, nil,
|
||||
func(h Hearing) { mu.Lock(); told = append(told, h); mu.Unlock() })
|
||||
hearing.Wait()
|
||||
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
|
||||
t.Fatalf("a refused hearing says %q", why)
|
||||
}
|
||||
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
|
||||
g.Write(delivery.Step{ID: "mvl0frv18d1973", Node: "g14", State: delivery.StepPassed})
|
||||
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hearing.Wait()
|
||||
mu.Lock()
|
||||
if hearing.Deaf() != "" || len(told) != 2 || told[0].Heard || !told[1].Heard || fol.calls() != 2 {
|
||||
t.Fatalf("after an ask: deaf %q, told %+v, followed %d times", hearing.Deaf(), told, fol.calls())
|
||||
}
|
||||
mu.Unlock()
|
||||
fol.heard(GivenUp{AskID: "mvl0frv18d1974", Deliveries: 5})
|
||||
step, err = k.Wait(context.Background(), goAsk("mvl0frv18d1974"))
|
||||
if err != nil || !IsLost(step) {
|
||||
t.Fatalf("an ask heard given up after the hearing came back: %+v, %v", step, err)
|
||||
}
|
||||
// The lost check of 2026-10-28: a step whose own watch went deaf is read at the next nudge (a step of its build module
|
||||
// ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again.
|
||||
d := &FakeAsks{}
|
||||
heard := &HeardEnds{Inner: deafWatch{}, Read: d}
|
||||
k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute}
|
||||
d.Write(delivery.Step{ID: "mvl0frv18d1980", Node: "shanks", State: delivery.StepPassed})
|
||||
got := make(chan delivery.Step, 1)
|
||||
go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }()
|
||||
for heard.Waiting() == 0 {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
heard.Nudge(NodeBuildModule)
|
||||
select {
|
||||
case s := <-got:
|
||||
t.Fatalf("a nudge of another build module answered %+v", s)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
heard.Nudge(GoBuildModule)
|
||||
if s := <-got; s.State != delivery.StepPassed || s.Node != "shanks" {
|
||||
t.Fatalf("a step its watch never handed over: %+v", s)
|
||||
}
|
||||
closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d}
|
||||
if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != delivery.StepPassed {
|
||||
t.Fatalf("a watch that closed once: %+v, %v", s, err)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
h *Hearing
|
||||
want string
|
||||
}{{nil, health.StateUnhealthy}, {&told[0], health.StateUnhealthy}, {&told[1], health.StateHealthy}} {
|
||||
var raw []byte
|
||||
if row.h != nil {
|
||||
raw, _ = json.Marshal(row.h)
|
||||
}
|
||||
if a := HearingHealth(raw); a.State != row.want || a.Why == "" {
|
||||
t.Errorf("health of %+v = %+v, want %s", row.h, a, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review (issue 734 path): an ask taken again by the same node (its build module replaced mid-run) is bounded from the
|
||||
// new run's start, not the first run's. The steps' own times stand for the clock: the first run started now, the second a
|
||||
// whole bound ago, so the wait ends at once on the second run's bound where the first run's would wait an hour; no case
|
||||
// waits on a pause a loaded machine can outlast (the check of 2026-11-02 22:35 tripped one of 60 ms). The context's
|
||||
// bound is reached only by a wait that kept the first run's start.
|
||||
func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
|
||||
f := &FakeAsks{}
|
||||
k := Asker{Publisher: f, Waiter: f, Bound: time.Hour}
|
||||
const id = "mvl0frv18d1962"
|
||||
now := time.Now()
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: now})
|
||||
f.Write(delivery.Step{ID: id, Node: "ace", State: delivery.StepRunning, Started: now.Add(-time.Hour)})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := k.Wait(ctx, goAsk(id)); err == nil || !strings.Contains(err.Error(), "on ace did not end within 1h0m0s of its start") {
|
||||
t.Fatalf("a redelivered run a bound old: %v, want its own bound to end the wait", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingFollower refuses the first `refuse` follows, as the bus does before the asker's grant is composed.
|
||||
type refusingFollower struct {
|
||||
mu sync.Mutex
|
||||
refuse int
|
||||
n int
|
||||
hand func(GivenUp)
|
||||
}
|
||||
|
||||
func (f *refusingFollower) Follow(_ context.Context, _ string, _ time.Time, heard func(GivenUp)) (func(), error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.n++
|
||||
if f.n <= f.refuse {
|
||||
return nil, errors.New("permissions violation")
|
||||
}
|
||||
f.hand = heard
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
func (f *refusingFollower) calls() int { f.mu.Lock(); defer f.mu.Unlock(); return f.n }
|
||||
|
||||
func (f *refusingFollower) heard(g GivenUp) { f.mu.Lock(); h := f.hand; f.mu.Unlock(); h(g) }
|
||||
|
||||
// deafWatch is a watch of an ask's step that stays open and hears nothing.
|
||||
type deafWatch struct{}
|
||||
|
||||
func (deafWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
|
||||
return make(chan delivery.Step), nil
|
||||
}
|
||||
|
||||
// closingWatch closes its first watch at once, and hands the later ones to inner.
|
||||
type closingWatch struct {
|
||||
inner AskWaiter
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (c *closingWatch) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if !c.closed {
|
||||
c.closed = true
|
||||
ch := make(chan delivery.Step)
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
return c.inner.WatchAsk(ctx, module, askID)
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
|
||||
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
|
||||
// part has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
|
||||
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
|
||||
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
|
||||
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
|
||||
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
|
||||
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
|
||||
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
|
||||
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
|
||||
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
|
||||
tc, none := delivery.AgreedToolchain(entries)
|
||||
if none != "" {
|
||||
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
|
||||
}
|
||||
nodes, err := choose(language, entries, places, qualifies, want)
|
||||
return nodes, tc, err
|
||||
}
|
||||
|
||||
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
|
||||
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
|
||||
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
return choose(language, entries, places, qualifies, want)
|
||||
}
|
||||
|
||||
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
if places == nil {
|
||||
places = &Places{}
|
||||
}
|
||||
nodes := make([]string, 0, len(entries))
|
||||
var reporting []string
|
||||
seen := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if seen[e.Node] {
|
||||
continue
|
||||
}
|
||||
seen[e.Node] = true
|
||||
reporting = append(reporting, e.Node)
|
||||
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the part off a node
|
||||
nodes = append(nodes, e.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
sort.Strings(reporting)
|
||||
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
|
||||
}
|
||||
return places.choose(nodes, want), nil
|
||||
}
|
||||
|
||||
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
|
||||
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
|
||||
// building again settles nothing; it waits for another fingerprint.
|
||||
const DigestsDifferPrefix = "digests differ:"
|
||||
|
||||
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
|
||||
// digest when they differ. One node's digest agrees with itself.
|
||||
func Agree(artifact string, byNode map[string]string) (string, error) {
|
||||
nodes := make([]string, 0, len(byNode))
|
||||
for n := range byNode {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var first string
|
||||
for i, n := range nodes {
|
||||
if i == 0 {
|
||||
first = byNode[n]
|
||||
continue
|
||||
}
|
||||
if byNode[n] != first {
|
||||
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
|
||||
for _, n := range nodes {
|
||||
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
|
||||
}
|
||||
return "", fmt.Errorf("%s", msg[:len(msg)-1])
|
||||
}
|
||||
}
|
||||
return first, nil
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
|
||||
func hex64(c byte) string { return strings.Repeat(string(c), 64) }
|
||||
|
||||
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
|
||||
var out []delivery.NodeToolchain
|
||||
for _, n := range nodes {
|
||||
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
|
||||
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
|
||||
}
|
||||
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
|
||||
t.Errorf("toolchain = %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
|
||||
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
|
||||
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
|
||||
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
|
||||
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
|
||||
}
|
||||
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
|
||||
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
|
||||
}
|
||||
}
|
||||
|
||||
func agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
|
||||
if err != nil || d != digest {
|
||||
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
|
||||
}
|
||||
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
|
||||
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
|
||||
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
|
||||
}
|
||||
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
|
||||
t.Errorf("one node: %q, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
|
||||
online := func(n string) bool { return n != "g14" } // a sleeping laptop
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
|
||||
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
|
||||
}
|
||||
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
|
||||
}
|
||||
if tc.Version != "go1.27.1" {
|
||||
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// placesRunning is Places with these steps counted as running, for a test of the choice.
|
||||
func placesRunning(running map[string]int) *Places {
|
||||
return &Places{running: running, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func choosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
|
||||
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
|
||||
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
|
||||
p.Capacity = map[string]int{"shanks": 3}
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The choice of the nodes a step runs on and the agreement of their answers, one table of cases (the test budget).
|
||||
func TestChoosingTheNodesOfAStep(t *testing.T) {
|
||||
t.Run("choose takes the agreed nodes with fewest running steps ties by name", chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName)
|
||||
t.Run("choose with one node answers one node", chooseWithOneNodeAnswersOneNode)
|
||||
t.Run("choose fails naming the language and the versions when no node qualifies", chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies)
|
||||
t.Run("agree needs equal digests and names both nodes when they differ", agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer)
|
||||
t.Run("choose skips nodes that are not online and fails naming them when none is", chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs)
|
||||
t.Run("choose prefers nodes below their capacity", choosePrefersNodesBelowTheirCapacity)
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
// Package buildrun holds the parts every module that runs steps on the build modules shares: the caller, the steps
|
||||
// watch and the runner of a long step (ADR 0469), the sets of presence, toolchains and healthy build modules, the
|
||||
// places a node offers by capacity, and the choice of build nodes (ADR 0478). The builder and the checker of the
|
||||
// delivery use them; they moved here from the builder's own package so neither keeps a copy (issue 1671, ADR 0484).
|
||||
package buildrun
|
||||
@@ -1,23 +0,0 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
@@ -1,29 +0,0 @@
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
@@ -1,56 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
|
||||
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
|
||||
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
|
||||
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
|
||||
// and from a build at once.
|
||||
type HaveSet struct {
|
||||
mu sync.Mutex
|
||||
healthy map[string]bool // by `<node>.<module>`
|
||||
}
|
||||
|
||||
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
|
||||
// (`<node>.node`) and a value that is not a have are left out.
|
||||
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return false
|
||||
}
|
||||
ok := false
|
||||
if !deleted {
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
||||
}
|
||||
k := node + "." + module
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.healthy[k] == ok {
|
||||
return false
|
||||
}
|
||||
if s.healthy == nil {
|
||||
s.healthy = map[string]bool{}
|
||||
}
|
||||
if ok {
|
||||
s.healthy[k] = true
|
||||
} else {
|
||||
delete(s.healthy, k)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Healthy is whether the module on the node was last seen running and healthy.
|
||||
func (s *HaveSet) Healthy(node, module string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.healthy[node+"."+module]
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A step's end is never only its own watch's to hear** (the lost check of 2026-10-28: mesh-catalog #553's verdict
|
||||
// stayed running after its step mvsgthyjed5786 had passed on shanks at 19:09:31Z; the watch of that one key, opened at
|
||||
// 18:46:05Z, heard neither the step's start nor its end and never closed, while a watch opened at 18:58:48Z heard its own
|
||||
// step end). HeardEnds stands between the asker (Asker) and the watch of each ask's step key: when it is
|
||||
// nudged, every ask it waits on reads its step keys now (a read, not the watch) and hands what it reads to its wait, so a
|
||||
// step that ended is judged even when its own watch went deaf. A watch that closes under a wait is opened again, at most
|
||||
// reopenTries times. It is nudged on an event (Nudge): any step of the same build module ending (one watch of each steps
|
||||
// state, HearEnds), a reconnect to the bus, and whatever else the asker hears (the checker: checker_ask_again).
|
||||
// It holds no timer. Every asker of the build seats waits through it (the checker and the builder, HearEnds).
|
||||
|
||||
// StepReader reads, now, every step a build module's steps state holds for an ask: its keys `*.<ask id>`.
|
||||
type StepReader interface {
|
||||
AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error)
|
||||
}
|
||||
|
||||
// reopenTries is how often a wait opens its ask's watch again after it closed, before the wait ends failed.
|
||||
const reopenTries = 3
|
||||
|
||||
// HeardEnds is an AskWaiter over Inner that also reads an ask's steps when nudged. Safe for concurrent use.
|
||||
type HeardEnds struct {
|
||||
Inner AskWaiter
|
||||
Read StepReader
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
waiting map[string]*waitedAsk // by ask id
|
||||
}
|
||||
|
||||
type waitedAsk struct {
|
||||
module string
|
||||
nudge chan struct{}
|
||||
}
|
||||
|
||||
func (h *HeardEnds) logf(format string, args ...any) {
|
||||
if h.Logf != nil {
|
||||
h.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's: what Inner's watch hands, and, at each nudge, what the ask's keys hold now.
|
||||
func (h *HeardEnds) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
in, err := h.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &waitedAsk{module: module, nudge: make(chan struct{}, 1)}
|
||||
h.mu.Lock()
|
||||
if h.waiting == nil {
|
||||
h.waiting = map[string]*waitedAsk{}
|
||||
}
|
||||
h.waiting[askID] = w
|
||||
h.mu.Unlock()
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
if h.waiting[askID] == w {
|
||||
delete(h.waiting, askID)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
reopened := 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-in:
|
||||
if !ok {
|
||||
if ctx.Err() != nil || reopened >= reopenTries {
|
||||
h.logf("the watch of the ask %s's step on %s closed %d times; its wait ends", askID, module, reopened+1)
|
||||
return
|
||||
}
|
||||
reopened++
|
||||
if in, err = h.Inner.WatchAsk(ctx, module, askID); err != nil {
|
||||
h.logf("the watch of the ask %s's step on %s closed and cannot be opened again, so its wait ends: %v", askID, module, err)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the ask %s's step on %s closed; opened again (%d of %d)", askID, module, reopened, reopenTries)
|
||||
continue
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case <-w.nudge:
|
||||
if h.Read == nil {
|
||||
continue
|
||||
}
|
||||
steps, err := h.Read.AskSteps(ctx, module, askID)
|
||||
if err != nil {
|
||||
h.logf("the steps of the ask %s on %s cannot be read: %v", askID, module, err)
|
||||
continue
|
||||
}
|
||||
for _, s := range steps {
|
||||
if s.State != delivery.StepRunning {
|
||||
h.logf("the ask %s's step on %s ended %s, read at a nudge: its watch had not handed it over", askID, s.Node, s.State)
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Nudge has every ask waited on of module read its steps now; module "" nudges every ask.
|
||||
func (h *HeardEnds) Nudge(module string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for _, w := range h.waiting {
|
||||
if module == "" || w.module == module {
|
||||
select {
|
||||
case w.nudge <- struct{}{}:
|
||||
default: // a nudge is pending already
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Waiting is how many asks are waited on now.
|
||||
func (h *HeardEnds) Waiting() int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return len(h.waiting)
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the keys `*.<ask id>` of the build module's steps state, read now.
|
||||
func (s JetStreamAsks) AskSteps(ctx context.Context, module, askID string) ([]delivery.Step, error) {
|
||||
pattern, err := delivery.AskStepWatch(askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := s.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
lister, err := kv.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []delivery.Step
|
||||
for key := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue // deleted between the list and the read
|
||||
}
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
out = append(out, step)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HearEnds is the HeardEnds every asker of the build seats waits through: over inner (the asks' own watches, or a
|
||||
// LostAskWaiter's inner), reading with asks, and nudged at each step end of each of modules (one watch of each steps state,
|
||||
// updates only; one that closes is opened again, and said and left after reopenTries closes in a row that heard nothing). The asker nudges it at a reconnect too (Nudge("")).
|
||||
func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules []string, logf func(string, ...any)) *HeardEnds {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf}
|
||||
for _, module := range modules {
|
||||
kv, err := asks.JS.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
|
||||
continue
|
||||
}
|
||||
go h.follow(ctx, kv, module)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) {
|
||||
silent := 0 // closes in a row having heard nothing
|
||||
for ctx.Err() == nil {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
h.logf("the steps of %s cannot be watched, so an ask whose own watch goes deaf is read only at a reconnect: %v", module, err)
|
||||
return
|
||||
}
|
||||
heard := false
|
||||
for e := range w.Updates() {
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
heard = true
|
||||
if step, err := delivery.DecodeStep(e.Value()); err == nil && step.State != delivery.StepRunning {
|
||||
h.Nudge(module)
|
||||
}
|
||||
}
|
||||
_ = w.Stop()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
h.Nudge(module) // a step may have ended while the watch was closing
|
||||
if heard {
|
||||
silent = 0
|
||||
} else if silent++; silent >= reopenTries {
|
||||
h.logf("the watch of the steps of %s closed %d times in a row having heard nothing, so it is left: an ask whose own watch goes deaf is read only at a reconnect", module, silent)
|
||||
return
|
||||
}
|
||||
h.logf("the watch of the steps of %s closed; opened again", module)
|
||||
}
|
||||
}
|
||||
|
||||
// AskSteps is StepReader's: the last step written for the ask, if any.
|
||||
func (f *FakeAsks) AskSteps(_ context.Context, _ string, askID string) ([]delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if s := f.steps[askID]; len(s) > 0 {
|
||||
return []delivery.Step{s[len(s)-1]}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,503 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// **An ask the build seat gave up on ends its step** (novox/hq issue 734). A build seat's worker offers an ask
|
||||
// delivery.AskMaxDeliver times; one handed out that often without an answer (a bus reload that drops every hand-out, a node
|
||||
// that dies taking it each time) is given up by the server, and the controller keeps a copy of it in DEAD_LETTERS under
|
||||
// mesh.events.dead.<seat stream>.<worker>, with the ask's message id (its ask id), how often it was handed out and when it
|
||||
// was given up. No build node writes a step for it, so an asker that waits for the step waits for ever: a try stays
|
||||
// running and holds a try place, a check's verdict stays running and its pull request waits.
|
||||
//
|
||||
// An asker of the build seats (the checker, the builder) hears those copies (DeadLetterHearing: an ordered consumer on
|
||||
// DEAD_LETTERS filtered to the build seats' streams, granted to a user of a seat pulled by kind, buscompose
|
||||
// DeadLetterReads) into DeadLetters, and LostAskWaiter ends the waiting step at once as failed on the node LostNode, its
|
||||
// reason saying LostWords; the asker decides what follows (the checker ends a try as error and asks a check again, the
|
||||
// builder asks its part of a build again).
|
||||
//
|
||||
// Belt and braces, where the asker says so in the context (NotTakenBound; the checker's tries): an ask no build node took
|
||||
// within a bound ends the same way, saying NotTakenWords. An ask waits in the stream as long as the backlog is otherwise
|
||||
// (ADR 0525 §9): a backlog is no fault.
|
||||
|
||||
// LostNode is the node a step ended for a lost ask names: no build node wrote it.
|
||||
const LostNode = "bus"
|
||||
|
||||
// The words of a step ended for its ask.
|
||||
const (
|
||||
LostWords = "its ask was given up by the build seat"
|
||||
NotTakenWords = "no build node took its ask"
|
||||
)
|
||||
|
||||
// The headers the controller keeps a given-up message's copy with in DEAD_LETTERS (mesh-controller internal/link
|
||||
// deadletters.go): the original message id (an ask's is its ask id), how often it was handed out, when it was given up.
|
||||
const (
|
||||
DeadHeaderMsgID = "Mesh-Dead-Msg-Id"
|
||||
DeadHeaderDeliveries = "Mesh-Dead-Deliveries"
|
||||
DeadHeaderGaveUp = "Mesh-Dead-Gave-Up"
|
||||
DeadHeaderStream = "Mesh-Dead-Stream"
|
||||
)
|
||||
|
||||
// DeadLettersStream is the stream the controller keeps given-up messages in, and DeadLetterFilter the subjects of one seat
|
||||
// stream's copies there (buscompose DeadLetterSubject).
|
||||
const DeadLettersStream = "DEAD_LETTERS"
|
||||
|
||||
// DeadLetterFilter is the subjects DEAD_LETTERS keeps a seat stream's given-up messages under.
|
||||
func DeadLetterFilter(stream string) string { return "mesh.events.dead." + stream + ".>" }
|
||||
|
||||
// SeatStream is the stream of a build seat's language (buscompose seatStreamName of node-build-<language>).
|
||||
func SeatStream(language string) string { return "SEAT_NODE_BUILD_" + strings.ToUpper(language) }
|
||||
|
||||
// GivenUp is one ask a build seat gave up on, as DEAD_LETTERS keeps it.
|
||||
type GivenUp struct {
|
||||
AskID string
|
||||
Stream string
|
||||
Deliveries int
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// GivenUpOf reads a DEAD_LETTERS copy's headers; false when it names no message id.
|
||||
func GivenUpOf(header map[string][]string) (GivenUp, bool) {
|
||||
get := func(k string) string {
|
||||
if v := header[k]; len(v) > 0 {
|
||||
return strings.TrimSpace(v[0])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
g := GivenUp{AskID: get(DeadHeaderMsgID), Stream: get(DeadHeaderStream)}
|
||||
if g.AskID == "" {
|
||||
return GivenUp{}, false
|
||||
}
|
||||
g.Deliveries, _ = strconv.Atoi(get(DeadHeaderDeliveries))
|
||||
g.At, _ = time.Parse(time.RFC3339Nano, get(DeadHeaderGaveUp))
|
||||
return g, true
|
||||
}
|
||||
|
||||
// Words are what a step ended for the given-up ask says.
|
||||
func (g GivenUp) Words() string {
|
||||
at := "at an unknown time"
|
||||
if !g.At.IsZero() {
|
||||
at = "at " + g.At.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return fmt.Sprintf("%s after %d deliveries %s (kept in %s); ask again", LostWords, g.Deliveries, at, DeadLettersStream)
|
||||
}
|
||||
|
||||
// keptGivenUp is how many given-up asks DeadLetters remembers for a waiter that starts after the copy was heard.
|
||||
const keptGivenUp = 1024
|
||||
|
||||
// DeadLetters holds the given-up asks heard, and hands each to the waiter of its ask. Safe for concurrent use.
|
||||
type DeadLetters struct {
|
||||
mu sync.Mutex
|
||||
heard map[string]GivenUp
|
||||
order []string
|
||||
watchers map[string][]chan GivenUp
|
||||
}
|
||||
|
||||
// Heard takes one given-up ask in.
|
||||
func (d *DeadLetters) Heard(g GivenUp) {
|
||||
d.mu.Lock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if _, ok := d.heard[g.AskID]; !ok {
|
||||
d.order = append(d.order, g.AskID)
|
||||
if len(d.order) > keptGivenUp {
|
||||
delete(d.heard, d.order[0])
|
||||
d.order = d.order[1:]
|
||||
}
|
||||
}
|
||||
d.heard[g.AskID] = g
|
||||
ws := d.watchers[g.AskID]
|
||||
delete(d.watchers, g.AskID)
|
||||
d.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
w <- g // buffered, one each
|
||||
}
|
||||
}
|
||||
|
||||
// watch answers a channel that gets the ask's given-up copy once (at once when it was heard already), and its cancel.
|
||||
func (d *DeadLetters) watch(askID string) (<-chan GivenUp, func()) {
|
||||
ch := make(chan GivenUp, 1)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.heard == nil {
|
||||
d.heard, d.watchers = map[string]GivenUp{}, map[string][]chan GivenUp{}
|
||||
}
|
||||
if g, ok := d.heard[askID]; ok {
|
||||
ch <- g
|
||||
return ch, func() {}
|
||||
}
|
||||
d.watchers[askID] = append(d.watchers[askID], ch)
|
||||
return ch, func() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
ws := d.watchers[askID]
|
||||
for i, w := range ws {
|
||||
if w == ch {
|
||||
d.watchers[askID] = append(ws[:i], ws[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(d.watchers[askID]) == 0 {
|
||||
delete(d.watchers, askID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type notTakenKey struct{}
|
||||
|
||||
// NotTakenBound says, in ctx, that an ask no build node took within bound ends its step (a try's, never a check's).
|
||||
func NotTakenBound(ctx context.Context, bound time.Duration) context.Context {
|
||||
return context.WithValue(ctx, notTakenKey{}, bound)
|
||||
}
|
||||
|
||||
func notTakenBoundOf(ctx context.Context) time.Duration {
|
||||
d, _ := ctx.Value(notTakenKey{}).(time.Duration)
|
||||
return d
|
||||
}
|
||||
|
||||
// LostAskWaiter is an AskWaiter that ends an ask's step when the build seat gave the ask up (DeadLetters), or, under
|
||||
// NotTakenBound, when no build node took it within the bound: it hands the asker a failed step whose reason says why, on
|
||||
// the node "bus". Every step the build module writes is handed on as it is.
|
||||
type LostAskWaiter struct {
|
||||
Inner AskWaiter
|
||||
Dead *DeadLetters
|
||||
// Hearing, when set, is asked to hear DEAD_LETTERS again each time an ask is waited for while it cannot: an ask put is
|
||||
// the event that makes hearing matter.
|
||||
Hearing interface {
|
||||
Deaf() string
|
||||
Again()
|
||||
}
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (w LostAskWaiter) now() time.Time {
|
||||
if w.Now != nil {
|
||||
return w.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// WatchAsk is AskWaiter's.
|
||||
func (w LostAskWaiter) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
||||
if w.Hearing != nil && w.Hearing.Deaf() != "" {
|
||||
w.Hearing.Again()
|
||||
}
|
||||
inner, err := w.Inner.WatchAsk(ctx, module, askID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dead <-chan GivenUp
|
||||
cancel := func() {}
|
||||
if w.Dead != nil {
|
||||
dead, cancel = w.Dead.watch(askID)
|
||||
}
|
||||
// The not-taken bound is a context's (ADR 0437 §8: no timer), ended when a node takes the ask.
|
||||
var notTaken <-chan struct{}
|
||||
bound := notTakenBoundOf(ctx)
|
||||
stopBound := func() {}
|
||||
if bound > 0 {
|
||||
var nt context.Context
|
||||
nt, stopBound = context.WithTimeout(context.Background(), bound)
|
||||
notTaken = nt.Done()
|
||||
}
|
||||
out := make(chan delivery.Step)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer cancel()
|
||||
defer func() { stopBound() }()
|
||||
send := func(s delivery.Step) bool {
|
||||
select {
|
||||
case out <- s:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
ended := func(reason string) delivery.Step {
|
||||
at := w.now().UTC()
|
||||
return delivery.Step{ID: askID, Node: LostNode, State: delivery.StepFailed, Reason: reason, Ended: at}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case s, ok := <-inner:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if notTaken != nil {
|
||||
stopBound() // a node took it: its step is bounded by the asker from its start
|
||||
notTaken = nil
|
||||
}
|
||||
if !send(s) {
|
||||
return
|
||||
}
|
||||
case g := <-dead:
|
||||
dead = nil
|
||||
send(ended(g.Words()))
|
||||
return
|
||||
case <-notTaken:
|
||||
send(ended(fmt.Sprintf("%s within %s of its put; it was lost or waits behind a backlog; ask again", NotTakenWords, bound)))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// IsLost says whether a step was ended by LostAskWaiter because the build seat gave its ask up (LostWords).
|
||||
func IsLost(s delivery.Step) bool { return s.Node == LostNode && strings.Contains(s.Reason, LostWords) }
|
||||
|
||||
// IsNotTaken says whether a step was ended by LostAskWaiter because no build node took its ask within the bound.
|
||||
func IsNotTaken(s delivery.Step) bool {
|
||||
return s.Node == LostNode && strings.Contains(s.Reason, NotTakenWords)
|
||||
}
|
||||
|
||||
// **The asker hears DEAD_LETTERS, tries again when it cannot, and says whether it hears** (issue 734). The read is
|
||||
// granted with the asker's seat traffic (buscompose DeadLetterReads); an asker started before its grant was composed is
|
||||
// refused, and an asker that only tried once would stay deaf until it starts again, silently. DeadLetterHearing tries in
|
||||
// the background, never holding the start up, and again on an event (Again): the asker's reconnect to the bus, and every
|
||||
// ask waited for while it is deaf (LostAskWaiter.Hearing); no timer. Each change of whether it hears is said (Said), so
|
||||
// the module's health tool says it: a deaf asker is not healthy.
|
||||
|
||||
// Hearing is what a DeadLetterHearing says: whether it hears every build seat stream it follows, and why not.
|
||||
type Hearing struct {
|
||||
Heard bool `json:"heard"`
|
||||
Streams []string `json:"streams"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HearingKey is the key a module keeps its Hearing under, in its own state (the checker's and the builder's hearing).
|
||||
const HearingKey = "dead-letters"
|
||||
|
||||
// DeadLetterFollower starts following one seat stream's copies in DEAD_LETTERS from since, handing each given-up ask to
|
||||
// heard; it answers the stop, or why it cannot.
|
||||
type DeadLetterFollower interface {
|
||||
Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error)
|
||||
}
|
||||
|
||||
// JetStreamDeadLetters is the bus side of a DeadLetterFollower: an ordered consumer on DEAD_LETTERS filtered to the
|
||||
// stream's copies, which resumes where it was after a reconnect and, reset before it heard any, starts at since again.
|
||||
type JetStreamDeadLetters struct{ JS jetstream.JetStream }
|
||||
|
||||
// Follow is DeadLetterFollower's.
|
||||
func (j JetStreamDeadLetters) Follow(ctx context.Context, stream string, since time.Time, heard func(GivenUp)) (func(), error) {
|
||||
cons, err := j.JS.OrderedConsumer(ctx, DeadLettersStream, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{DeadLetterFilter(stream)}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
if g, ok := GivenUpOf(m.Headers()); ok {
|
||||
heard(g)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cc.Stop, nil
|
||||
}
|
||||
|
||||
// DeadLetterHearing follows the build seats' given-up asks into Dead. Make it with NewDeadLetterHearing.
|
||||
type DeadLetterHearing struct {
|
||||
follower DeadLetterFollower
|
||||
streams []string
|
||||
dead *DeadLetters
|
||||
logf func(string, ...any)
|
||||
said func(Hearing)
|
||||
ctx context.Context
|
||||
since time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
stops map[string]func()
|
||||
why map[string]string
|
||||
trying bool
|
||||
again bool
|
||||
told *Hearing
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewDeadLetterHearing starts hearing the given-up asks of each language's build seat stream, in the background. said
|
||||
// is told each change of whether it hears (nil tells nothing); ctx ends it.
|
||||
func NewDeadLetterHearing(ctx context.Context, follower DeadLetterFollower, languages []string, dead *DeadLetters,
|
||||
logf func(string, ...any), said func(Hearing)) *DeadLetterHearing {
|
||||
h := &DeadLetterHearing{follower: follower, dead: dead, logf: logf, said: said, ctx: ctx, since: time.Now(),
|
||||
stops: map[string]func(){}, why: map[string]string{}}
|
||||
for _, l := range languages {
|
||||
s := SeatStream(l)
|
||||
h.streams = append(h.streams, s)
|
||||
h.why[s] = "not tried yet"
|
||||
}
|
||||
if h.logf == nil {
|
||||
h.logf = func(string, ...any) {}
|
||||
}
|
||||
h.Again()
|
||||
return h
|
||||
}
|
||||
|
||||
// Deaf says why a stream is not heard, or "" when every one is.
|
||||
func (h *DeadLetterHearing) Deaf() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.deafLocked()
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) deafLocked() string {
|
||||
var out []string
|
||||
for _, s := range h.streams {
|
||||
if why, deaf := h.why[s]; deaf {
|
||||
out = append(out, s+": "+why)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
return DeadLettersStream + " is not heard, so an ask a build seat gives up on is waited for until its bound: " +
|
||||
strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
// Again tries, in the background, every stream not heard yet; a call while it tries makes it try once more after.
|
||||
func (h *DeadLetterHearing) Again() {
|
||||
h.mu.Lock()
|
||||
if h.trying {
|
||||
h.again = true
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.trying = true
|
||||
h.mu.Unlock()
|
||||
h.wg.Add(1)
|
||||
go func() {
|
||||
defer h.wg.Done()
|
||||
for {
|
||||
h.try()
|
||||
h.mu.Lock()
|
||||
if !h.again || h.ctx.Err() != nil {
|
||||
h.trying = false
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.again = false
|
||||
h.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait returns when no try runs.
|
||||
func (h *DeadLetterHearing) Wait() { h.wg.Wait() }
|
||||
|
||||
// Stop ends every stream's following.
|
||||
func (h *DeadLetterHearing) Stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for s, stop := range h.stops {
|
||||
stop()
|
||||
delete(h.stops, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DeadLetterHearing) try() {
|
||||
for _, s := range h.streams {
|
||||
h.mu.Lock()
|
||||
_, deaf := h.why[s]
|
||||
h.mu.Unlock()
|
||||
if !deaf || h.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
stream := s
|
||||
stop, err := h.follower.Follow(h.ctx, stream, h.since, func(g GivenUp) {
|
||||
h.logf("the ask %s on %s was given up after %d deliveries at %s: the wait for its step ends", g.AskID, g.Stream,
|
||||
g.Deliveries, g.At.Format(time.RFC3339))
|
||||
h.dead.Heard(g)
|
||||
})
|
||||
h.mu.Lock()
|
||||
if err != nil {
|
||||
h.why[stream] = err.Error()
|
||||
h.logf("the asks %s gives up on are not heard (%s of %s); tried again at the next reconnect or ask: %v",
|
||||
stream, DeadLetterFilter(stream), DeadLettersStream, err)
|
||||
} else {
|
||||
delete(h.why, stream)
|
||||
h.stops[stream] = stop
|
||||
h.logf("the asks %s gives up on are heard from %s", stream, DeadLettersStream)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
h.tell()
|
||||
}
|
||||
|
||||
// tell says the hearing when it changed.
|
||||
func (h *DeadLetterHearing) tell() {
|
||||
h.mu.Lock()
|
||||
why := h.deafLocked()
|
||||
now := Hearing{Heard: why == "", Streams: append([]string(nil), h.streams...), Why: why, At: time.Now().UTC()}
|
||||
same := h.told != nil && h.told.Heard == now.Heard && h.told.Why == now.Why
|
||||
if !same {
|
||||
h.told = &now
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if !same && h.said != nil {
|
||||
h.said(now)
|
||||
}
|
||||
}
|
||||
|
||||
// HearingState is the name of the state an asker keeps its Hearing in (`<module>_hearing`, key HearingKey), which its
|
||||
// health tool reads.
|
||||
const HearingState = "hearing"
|
||||
|
||||
// HearingKeeper is where a Hearing is kept: a jetstream.KeyValue, or a process.AwaitedBucket for a state that may not be
|
||||
// on the bus at the part's start (issue 1790), which keeps the latest until it appears and answers process.ErrNotYet.
|
||||
type HearingKeeper interface {
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// SayHearing is a DeadLetterHearing's said that keeps each Hearing in kv under HearingKey; nil kv keeps nothing and
|
||||
// says so once. A write kv keeps for later (process.ErrNotYet) is not an error: the awaited bucket said it waits.
|
||||
func SayHearing(ctx context.Context, kv HearingKeeper, logf func(string, ...any)) func(Hearing) {
|
||||
if kv == nil {
|
||||
if logf != nil {
|
||||
logf("whether %s is heard is not kept: the state %s is not there, so the health tool cannot say it", DeadLettersStream, HearingState)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return func(h Hearing) {
|
||||
data, err := json.Marshal(h)
|
||||
if err == nil {
|
||||
_, err = kv.Put(ctx, HearingKey, data)
|
||||
}
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) && logf != nil {
|
||||
logf("whether %s is heard could not be kept: %v", DeadLettersStream, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HearingHealth is the health answer of an asker's kept Hearing (raw, empty when none is kept): healthy while it hears
|
||||
// DEAD_LETTERS, unhealthy saying why while it does not or has not said.
|
||||
func HearingHealth(raw []byte) health.Answer {
|
||||
if len(raw) == 0 {
|
||||
return health.Unhealthy("it has not said whether it hears " + DeadLettersStream + " (issue 734): it is starting, or its state " + HearingState + " is not there")
|
||||
}
|
||||
var h Hearing
|
||||
if err := json.Unmarshal(raw, &h); err != nil {
|
||||
return health.Unhealthy("what it keeps of hearing " + DeadLettersStream + " is not a hearing: " + err.Error())
|
||||
}
|
||||
if !h.Heard {
|
||||
return health.Unhealthy(h.Why + " (since " + h.At.Format(time.RFC3339) + "; tried again at its next reconnect or ask)")
|
||||
}
|
||||
return health.Healthy("it hears " + DeadLettersStream + " for " + strings.Join(h.Streams, ", "))
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
// The build modules the mesh runs, by the language each builds (ADR 0469).
|
||||
const (
|
||||
GoBuildModule = "mesh-build-go"
|
||||
NodeBuildModule = "mesh-build-node"
|
||||
)
|
||||
@@ -1,145 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
|
||||
const DefaultCapacity = 1
|
||||
|
||||
// Places counts the build steps a part runs on each node and holds a lane back while its node is at its
|
||||
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
|
||||
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
|
||||
// reserving a place on it at the moment of the choice (ChooseNodes), so that builds choosing at once see each other's
|
||||
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
|
||||
// other lanes, not by a timer: ending a lane wakes the waiting ones.
|
||||
type Places struct {
|
||||
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
|
||||
Capacity map[string]int
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]int // lanes holding a place
|
||||
reserved map[string]int // lanes chosen for the node that do not hold a place yet
|
||||
changed chan struct{} // closed, and replaced, each time a lane ends
|
||||
}
|
||||
|
||||
// CapacityOf is how many steps the part runs on the node at once: DefaultCapacity for a node the setting does not
|
||||
// name, and 0 for one it names so, where the part never builds.
|
||||
func (p *Places) CapacityOf(node string) int {
|
||||
if n, ok := p.Capacity[node]; ok && n >= 0 {
|
||||
return n
|
||||
}
|
||||
return DefaultCapacity
|
||||
}
|
||||
|
||||
// Running is the steps in flight now, by node.
|
||||
func (p *Places) Running() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.running))
|
||||
for n, c := range p.running {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Reserved is the lanes chosen for a node that hold no place yet, by node.
|
||||
func (p *Places) Reserved() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.reserved))
|
||||
for n, c := range p.reserved {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Places) init() {
|
||||
if p.running == nil {
|
||||
p.running = map[string]int{}
|
||||
}
|
||||
if p.reserved == nil {
|
||||
p.reserved = map[string]int{}
|
||||
}
|
||||
if p.changed == nil {
|
||||
p.changed = make(chan struct{})
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
|
||||
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
|
||||
// gives up the reservation too.
|
||||
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
|
||||
for {
|
||||
p.mu.Lock()
|
||||
p.init()
|
||||
if p.running[node] < p.CapacityOf(node) {
|
||||
p.running[node]++
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
p.mu.Lock()
|
||||
p.running[node]--
|
||||
close(p.changed)
|
||||
p.changed = make(chan struct{})
|
||||
p.mu.Unlock()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
wait := p.changed
|
||||
p.mu.Unlock()
|
||||
select {
|
||||
case <-wait:
|
||||
case <-ctx.Done():
|
||||
p.Cancel(node)
|
||||
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel gives up a place reserved for the node that will not be taken.
|
||||
func (p *Places) Cancel(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
}
|
||||
|
||||
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
|
||||
// them, then by name), keeps the first want and reserves a place on each, all at once.
|
||||
func (p *Places) choose(nodes []string, want int) []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
load := func(n string) int { return p.running[n] + p.reserved[n] }
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
|
||||
if ri != rj {
|
||||
return ri
|
||||
}
|
||||
if load(nodes[i]) != load(nodes[j]) {
|
||||
return load(nodes[i]) < load(nodes[j])
|
||||
}
|
||||
return nodes[i] < nodes[j]
|
||||
})
|
||||
if len(nodes) > want {
|
||||
nodes = nodes[:want]
|
||||
}
|
||||
for _, n := range nodes {
|
||||
p.reserved[n]++
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func acquireAsync(p *Places, node string) chan func() {
|
||||
got := make(chan func(), 1)
|
||||
go func() {
|
||||
release, err := p.Acquire(context.Background(), node)
|
||||
if err != nil {
|
||||
close(got)
|
||||
return
|
||||
}
|
||||
got <- release
|
||||
}()
|
||||
return got
|
||||
}
|
||||
|
||||
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"ace": 2}}
|
||||
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
|
||||
ra, rb := <-a, <-b
|
||||
third := acquireAsync(p, "ace")
|
||||
select {
|
||||
case <-third:
|
||||
t.Fatal("a third step started on a node of capacity 2 with two running")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
if p.Running()["ace"] != 2 {
|
||||
t.Errorf("running = %v, want 2 on ace", p.Running())
|
||||
}
|
||||
ra()
|
||||
select {
|
||||
case r := <-third:
|
||||
r()
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the third step did not start when one ended")
|
||||
}
|
||||
rb()
|
||||
}
|
||||
|
||||
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3}}
|
||||
first := <-acquireAsync(p, "g14")
|
||||
second := acquireAsync(p, "g14")
|
||||
select {
|
||||
case <-second:
|
||||
t.Fatal("two steps on a node without a setting")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
first()
|
||||
r := <-second
|
||||
r()
|
||||
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
|
||||
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
|
||||
p := &Places{}
|
||||
r := <-acquireAsync(p, "ace")
|
||||
defer r()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Error("Acquire returned no error for a cancelled wait")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a cancelled wait did not return")
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
|
||||
func TestACapacityOfZeroKeepsThePartOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3, "ace": 2, "g14": 0, "novox": 0}}
|
||||
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
|
||||
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
|
||||
}
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
|
||||
for i := 0; i < 6; i++ {
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n == "g14" || n == "novox" {
|
||||
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
|
||||
_, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "capacity") {
|
||||
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
|
||||
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
|
||||
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
|
||||
type PresenceSet struct {
|
||||
mu sync.Mutex
|
||||
online map[string]bool
|
||||
}
|
||||
|
||||
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
|
||||
// value that is not a presence are left out.
|
||||
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, part, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || part != "" {
|
||||
return false
|
||||
}
|
||||
on := false
|
||||
if !deleted {
|
||||
p, err := delivery.DecodePresence(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
on = p.Online
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.online[node] == on {
|
||||
return false
|
||||
}
|
||||
if s.online == nil {
|
||||
s.online = map[string]bool{}
|
||||
}
|
||||
if on {
|
||||
s.online[node] = true
|
||||
} else {
|
||||
delete(s.online, node)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Online is whether the watchman last saw the node online.
|
||||
func (s *PresenceSet) Online(node string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.online[node]
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func presenceValue(t *testing.T, online bool) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
|
||||
var s PresenceSet
|
||||
if s.Online("ace") {
|
||||
t.Error("a node the watchman never wrote is online")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
|
||||
t.Error("ace online was not taken in")
|
||||
}
|
||||
if s.Put("ace", presenceValue(t, true), false) {
|
||||
t.Error("the same presence again counted as a change")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
|
||||
t.Error("ace going offline was not taken in")
|
||||
}
|
||||
s.Put("g14", presenceValue(t, true), false)
|
||||
if !s.Put("g14", nil, true) || s.Online("g14") {
|
||||
t.Error("a deleted key is still online")
|
||||
}
|
||||
// A part's instance (part.<part>.<instance>) is not a node.
|
||||
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
|
||||
t.Error("a part's key was taken for a node")
|
||||
}
|
||||
if s.Put("novox", []byte("junk"), false) {
|
||||
t.Error("junk counted")
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the
|
||||
// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left
|
||||
// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// ToolchainSet): an entry is passed on only while the placement (the seat mesh-placement's state `placement`, held by
|
||||
// the assigner, delivery.PlacementBucket, read by delivery.Placed; ADR 0567, ADR 0549 §3) places its build module on
|
||||
// its node, and only once it was written since that placement; it is passed on as deleted otherwise, said once, and
|
||||
// passed on when its node writes it again. A report written before the placement is one an earlier run of the module
|
||||
// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23
|
||||
// back into the agreement before the module ran there). Until the placement is settled, and while no node's placement
|
||||
// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed
|
||||
// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465,
|
||||
// and it stays.
|
||||
//
|
||||
// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's
|
||||
// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6).
|
||||
type ToolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
placed *delivery.Placed
|
||||
known bool // the placement is settled
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
// OnPlaced is told, outside the gate's lock, once the placement's history is read and after each revision of it
|
||||
// FollowPlacement hears from then on (the counted nodes, Nodes, may have changed). Nil tells nobody.
|
||||
OnPlaced func()
|
||||
}
|
||||
|
||||
// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what
|
||||
// it leaves out.
|
||||
func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate {
|
||||
g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: delivery.NewPlaced(),
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Placement takes one revision of the seat mesh-placement's placement in (at is when the bus wrote it) and answers whether what it
|
||||
// passed on changed. A value that does not read is said, and places nothing on its node.
|
||||
func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was := g.placed.Seeded()
|
||||
node, changed, err := g.placed.Put(key, value, deleted, at)
|
||||
if err != nil {
|
||||
g.logf("%s: %v (issue 1940)", delivery.PlacementBucket, err)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
if was != g.placed.Seeded() {
|
||||
// The first placement came, or the last went: every report's standing changed.
|
||||
return g.all()
|
||||
}
|
||||
return g.again(node, changed)
|
||||
}
|
||||
|
||||
// Nodes are the nodes the placement places any module on, sorted: the nodes the mesh counts (a node leaves the count
|
||||
// when the placement no longer names it; ADR 0624 §1). None until the placement is read.
|
||||
func (g *ToolchainGate) Nodes() []string {
|
||||
var out []string
|
||||
for node, modules := range g.placed.ByNode() {
|
||||
if len(modules) > 0 {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's
|
||||
// placement is held. It answers whether what it passed on changed.
|
||||
func (g *ToolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
return g.all()
|
||||
}
|
||||
|
||||
// all passes on every report again. The caller holds mu.
|
||||
func (g *ToolchainGate) all() bool {
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// again passes on every report of node by a build module among modules. The caller holds mu.
|
||||
func (g *ToolchainGate) again(node string, modules []string) bool {
|
||||
changed := false
|
||||
for _, module := range modules {
|
||||
if !g.modules[module] {
|
||||
continue
|
||||
}
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the placement is not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *ToolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known && g.placed.Seeded() {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its
|
||||
// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *ToolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, placed := g.placed.Since(node, module)
|
||||
if !placed {
|
||||
return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FollowPlacement reads the seat mesh-placement's placement with its history into gate, says it settled, and follows it; changed
|
||||
// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part
|
||||
// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that
|
||||
// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel.
|
||||
func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *ToolchainGate, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, delivery.PlacementBucket)
|
||||
if err != nil {
|
||||
logf("the state %s cannot be read, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
logf("the state %s cannot be watched, so every node's toolchain report counts, a build node's the catalogue does not place too (issue 1940): %v", delivery.PlacementBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
put := func(e jetstream.KeyValueEntry) bool {
|
||||
return gate.Placement(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut, e.Created())
|
||||
}
|
||||
for settled := false; !settled; {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = w.Stop()
|
||||
return errs, ctx.Err()
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errs, errors.New("the watch of " + delivery.PlacementBucket + " ended before it settled")
|
||||
}
|
||||
if e == nil {
|
||||
settled = true
|
||||
continue
|
||||
}
|
||||
put(e)
|
||||
}
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
if gate.OnPlaced != nil {
|
||||
gate.OnPlaced()
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs <- ctx.Err()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
errs <- errors.New("the watch of " + delivery.PlacementBucket + " ended")
|
||||
return
|
||||
}
|
||||
if e != nil && put(e) {
|
||||
changed(ctx)
|
||||
}
|
||||
if e != nil && gate.OnPlaced != nil {
|
||||
gate.OnPlaced()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
// placing is a value of the state `placement` placing modules on node.
|
||||
func placing(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
return key, values[key]
|
||||
}
|
||||
|
||||
// placingLegacy is placing in the form before, `placement.<node>` (research 072 row 37).
|
||||
func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) {
|
||||
t.Helper()
|
||||
p := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
key, _ := delivery.LegacyPlacementKey(node)
|
||||
value, err := delivery.EncodePlacement(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while
|
||||
// the placement places its build module on its node, and only once written since; before the placement is settled, and
|
||||
// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left
|
||||
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
|
||||
// back before the module ran there.
|
||||
func TestToolchainGate(t *testing.T) {
|
||||
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
|
||||
three := []string{"g14", "novox", "shanks"}
|
||||
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
|
||||
set := &ToolchainSet{}
|
||||
said := &[]string{}
|
||||
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range three {
|
||||
if seeded {
|
||||
key, value := placing(t, node, NodeBuildModule)
|
||||
gate.Placement(key, value, false, old.Add(-time.Hour))
|
||||
}
|
||||
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
}
|
||||
return gate, set, said
|
||||
}
|
||||
|
||||
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
|
||||
t.Fatalf("before the placement is settled every report counts; got %+v", a)
|
||||
}
|
||||
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
|
||||
gate.Placement(k, v, false, old)
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
|
||||
t.Fatalf("ace's report is said left out once; said %q", *said)
|
||||
}
|
||||
if gate.Toolchain(NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node not placed changed what was passed on")
|
||||
}
|
||||
// Placed on ace with a report written since: it counts, and the builders really disagree.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
gate.Placement(k, v, false, old.Add(-time.Minute))
|
||||
if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
|
||||
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
|
||||
}
|
||||
// Taken off ace again (its key deleted): its report goes again.
|
||||
if !gate.Placement(k, nil, true, placed) {
|
||||
t.Fatal("taking ace's placement away passed nothing on")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is placed or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1", placed)
|
||||
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
|
||||
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
|
||||
}
|
||||
})
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"a report left before the placement", old, false},
|
||||
{"a report written since the placement", placed.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gate, set, said := start(t, true)
|
||||
k, v := placing(t, "ace") // ace's key, first heard placing nothing
|
||||
gate.Placement(k, v, false, old.Add(-time.Hour))
|
||||
gate.Settled()
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed) {
|
||||
t.Fatal("a placement with no report passed something on")
|
||||
}
|
||||
key, value := gateReport(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// ace's placement written again with another module beside: its standing does not move, nothing is said again.
|
||||
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
||||
if gate.Placement(k, v, false, placed.Add(time.Minute)) {
|
||||
t.Fatal("the same placement of mesh-build-node changed what was passed on")
|
||||
}
|
||||
if c.counts {
|
||||
return
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
|
||||
}
|
||||
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", *said)
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
|
||||
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("no placement held: every report counts", func(t *testing.T) {
|
||||
gate, set, _ := start(t, false)
|
||||
key, value := gateReport(t, "ace", "v26.12.0", old)
|
||||
gate.Toolchain(NodeBuildModule, key, value, false)
|
||||
gate.Settled()
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("before the first placement file every report counts; got %d", n)
|
||||
}
|
||||
k, v := placing(t, "g14", NodeBuildModule)
|
||||
if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
|
||||
t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript")))
|
||||
}
|
||||
if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
|
||||
t.Fatal("the last placement gone: every report counts again")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
|
||||
p := delivery.NewPlaced()
|
||||
// ace in the form before, then converted: its module stands since before, the legacy key read no more.
|
||||
k, v := placingLegacy(t, "ace", NodeBuildModule)
|
||||
p.Put(k, v, false, old)
|
||||
k, v = placing(t, "ace", NodeBuildModule)
|
||||
if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil {
|
||||
t.Errorf("the conversion of ace's key changed %v, %v", changed, err)
|
||||
}
|
||||
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
|
||||
t.Errorf("a module key says which modules stand: %v", changed)
|
||||
}
|
||||
if k, v := placingLegacy(t, "ace"); true {
|
||||
if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 {
|
||||
t.Errorf("the legacy key is read after the node key: %v", changed)
|
||||
}
|
||||
}
|
||||
k2, v2 := placing(t, "g14")
|
||||
p.Put(k2, v2, false, old)
|
||||
k2, v2 = placing(t, "g14", NodeBuildModule)
|
||||
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
|
||||
t.Errorf("a module placed says changed %v", changed)
|
||||
}
|
||||
if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil {
|
||||
t.Error("a value that does not read was not said")
|
||||
}
|
||||
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
|
||||
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
|
||||
}
|
||||
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
|
||||
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
|
||||
}
|
||||
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
|
||||
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement")
|
||||
gate, _, _ := start(t, false)
|
||||
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
|
||||
if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
|
||||
t.Errorf("a refused placement stopped the start: %v", err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
|
||||
type Caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
|
||||
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
|
||||
// ends or ctx does.
|
||||
type Steps interface {
|
||||
Wait(ctx context.Context, module, key string) (delivery.Step, error)
|
||||
}
|
||||
|
||||
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
|
||||
// key when the step ends (ADR 0469 clause 2).
|
||||
type Runner struct {
|
||||
Caller Caller
|
||||
Steps Steps
|
||||
// Bound is how long a step may run before the builder stops waiting for it.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
|
||||
// failed or was refused is an error naming the node, the state and the reason.
|
||||
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
|
||||
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
var answer struct {
|
||||
Step string `json:"step"`
|
||||
Key string `json:"key"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
|
||||
}
|
||||
if answer.Key == "" {
|
||||
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
|
||||
}
|
||||
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
|
||||
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
|
||||
}
|
||||
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
|
||||
defer cancel()
|
||||
step, err := r.Steps.Wait(wait, module, answer.Key)
|
||||
if err != nil {
|
||||
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
|
||||
err = cause
|
||||
}
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
|
||||
}
|
||||
return step.Result, nil
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
type call struct {
|
||||
node, key string
|
||||
body any
|
||||
}
|
||||
|
||||
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
|
||||
type fakeCaller struct {
|
||||
calls []call
|
||||
answer string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
f.calls = append(f.calls, call{node, key, body})
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return json.RawMessage(f.answer), nil
|
||||
}
|
||||
|
||||
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
|
||||
type fakeSteps struct {
|
||||
mu sync.Mutex
|
||||
steps map[string]delivery.Step
|
||||
// during, when set, is called as a wait starts and the function it answers as the wait ends.
|
||||
during func(key string) func()
|
||||
}
|
||||
|
||||
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
s, ok := f.steps[module+"/"+key]
|
||||
during := f.during
|
||||
f.mu.Unlock()
|
||||
if ok {
|
||||
if during != nil {
|
||||
defer during(key)()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return delivery.Step{}, context.Cause(ctx)
|
||||
}
|
||||
|
||||
func stepAt(node, id, state, reason string, result string) delivery.Step {
|
||||
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
|
||||
if result != "" {
|
||||
s.Result = json.RawMessage(result)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func runCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
|
||||
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
|
||||
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
|
||||
}
|
||||
var fr struct{ Workspace string }
|
||||
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
|
||||
t.Errorf("result = %s (%v), want the step's result", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func runFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
|
||||
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
|
||||
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
|
||||
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
|
||||
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
|
||||
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if err == nil {
|
||||
t.Errorf("answer %s: no error", answer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
|
||||
boom := errors.New("no responders")
|
||||
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("error = %v, want the call's error naming the node", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A step run on its node and read from its steps key, one table of cases (the test budget).
|
||||
func TestRunningAStepOnANode(t *testing.T) {
|
||||
t.Run("run calls the tool on the node and reads the steps key", runCallsTheToolOnTheNodeAndReadsTheStepsKey)
|
||||
t.Run("run fails on a failed or refused step naming its reason", runFailsOnAFailedOrRefusedStepNamingItsReason)
|
||||
t.Run("run stops waiting at the bound", runStopsWaitingAtTheBound)
|
||||
t.Run("run refuses an answer with no key and a key of another node", runRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode)
|
||||
t.Run("run returns the calls own error", runReturnsTheCallsOwnError)
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
|
||||
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
|
||||
|
||||
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
|
||||
// put them in. Safe for use from the watches and from a build at once.
|
||||
type ToolchainSet struct {
|
||||
mu sync.Mutex
|
||||
by map[string]map[string]delivery.Toolchain // language -> node -> entry
|
||||
}
|
||||
|
||||
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
|
||||
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
|
||||
// A key or a value that is not an entry of the module's language is left out.
|
||||
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
|
||||
language, ok := LanguageOf[module]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
node, keyLanguage, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || keyLanguage != language {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := s.by[language][node]; !had {
|
||||
return false
|
||||
}
|
||||
delete(s.by[language], node)
|
||||
return true
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err != nil || tc.Language != language {
|
||||
return false
|
||||
}
|
||||
if s.by == nil {
|
||||
s.by = map[string]map[string]delivery.Toolchain{}
|
||||
}
|
||||
if s.by[language] == nil {
|
||||
s.by[language] = map[string]delivery.Toolchain{}
|
||||
}
|
||||
cur, had := s.by[language][node]
|
||||
s.by[language][node] = tc
|
||||
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
|
||||
}
|
||||
|
||||
// Entries are the nodes' entries of a language, in no particular order.
|
||||
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []delivery.NodeToolchain
|
||||
for node, tc := range s.by[language] {
|
||||
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package buildrun
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func tcValue(t *testing.T, language, version string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
|
||||
var s ToolchainSet
|
||||
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("a first entry did not count as a change")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("the same entry again counted as a change")
|
||||
}
|
||||
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
|
||||
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
if got := s.Entries("go"); len(got) != 2 {
|
||||
t.Errorf("go entries = %+v, want two", got)
|
||||
}
|
||||
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
|
||||
t.Errorf("typescript entries = %+v", got)
|
||||
}
|
||||
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
|
||||
t.Error("a delete did not remove the node's entry")
|
||||
}
|
||||
// What is not a toolchain of the module's language is left out.
|
||||
s = ToolchainSet{}
|
||||
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
|
||||
t.Error("a typescript toolchain in the go build module's bucket counted")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
|
||||
t.Error("garbage or an unknown module counted")
|
||||
}
|
||||
if len(s.Entries("go")) != 0 {
|
||||
t.Error("an entry was kept")
|
||||
}
|
||||
}
|
||||
@@ -7,132 +7,31 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// EngineReadsTool lists, asks again and drops what each counted node's engine reads, as the builder keeps it.
|
||||
const EngineReadsTool = "builder_engine_reads"
|
||||
|
||||
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
// tags writes a tag-asks value and waits for the builder's answer; ta reads tag-asks.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader,
|
||||
tags queueAsker, ta, fw, of versions.Reader) []stdio.Tool {
|
||||
// the builder's read.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
// handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error.
|
||||
handAct := func(key string, value []byte, err error) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Refused {
|
||||
return nil, errors.New(a.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil
|
||||
}
|
||||
tagAct := func(key string, value []byte, err error) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := tags(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return versions.DecodeTagAsk(raw)
|
||||
}
|
||||
nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{
|
||||
"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>, as builder_versions lists it"},
|
||||
"fp": map[string]any{"type": "string", "description": "the fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}}
|
||||
branch := func(head bool) map[string]any {
|
||||
req := []string{"repository", "branch", "why"}
|
||||
props := map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"branch": map[string]any{"type": "string", "description": "the branch; its cleaned name is the tag"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}
|
||||
if head {
|
||||
req = append(req, "head")
|
||||
props["head"] = map[string]any{"type": "string", "description": "the commit the tagged version is built from"}
|
||||
}
|
||||
return map[string]any{"type": "object", "required": req, "properties": props}
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return handAct(build.NewPublishAgain(str(args, "name"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()))
|
||||
}},
|
||||
{Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription,
|
||||
Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{
|
||||
"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return handAct(build.NewFloorAgain(str(args, "name"), str(args, "why"), stdio.CallerOf(args), now()))
|
||||
}},
|
||||
{Name: versions.TagTool, Description: versions.TagDescription, Input: branch(true),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return tagAct(versions.NewTagAsk(str(args, "repository"), str(args, "branch"), str(args, "head"), str(args, "why"), stdio.CallerOf(args), now()))
|
||||
}},
|
||||
{Name: versions.UntagTool, Description: versions.UntagDescription, Input: branch(false),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return tagAct(versions.NewUntag(str(args, "repository"), str(args, "branch"), str(args, "why"), stdio.CallerOf(args), now()))
|
||||
}},
|
||||
{Name: versions.TagAsksTool, Description: versions.TagAsksDescription,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "only this repository's asks, <owner>/<name>"}}},
|
||||
Run: func(args map[string]any) (any, error) { return versions.AnswerTagAsks(ta, str(args, "repository")) }},
|
||||
{Name: versions.OldFormTool, Description: versions.OldFormDescription,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"drop": map[string]any{"type": "string", "description": "a retired node to drop from the count by hand"},
|
||||
"why": map[string]any{"type": "string", "description": "with drop: why, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if node := str(args, "drop"); node != "" {
|
||||
return handAct(build.NewOldFormDrop(node, str(args, "why"), stdio.CallerOf(args), now()))
|
||||
}
|
||||
return versions.AnswerOldForm(of)
|
||||
}},
|
||||
{Name: versions.FloorWaitsTool, Description: versions.FloorWaitsDescription,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return versions.AnswerFloorWaits(fw) }},
|
||||
{Name: versions.VersionsTool, Description: versions.VersionsDescription,
|
||||
Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
|
||||
"name": map[string]any{"type": "string", "description": "the package, image, bundle or archive name"},
|
||||
"version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"},
|
||||
"digest": map[string]any{"type": "string", "description": "with version: the digest the holder received"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
|
||||
}},
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
||||
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
@@ -156,84 +55,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.BuildAgainTool,
|
||||
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
||||
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
||||
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
||||
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
||||
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
||||
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
if r.Revision == 0 && !r.Pending {
|
||||
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
||||
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
||||
}},
|
||||
{Name: delivery.BuildsSinceTool,
|
||||
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
|
||||
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
|
||||
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
|
||||
"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
|
||||
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
|
||||
}
|
||||
return build.BuildsSince(since, at)
|
||||
}},
|
||||
{Name: build.AnnounceAgainTool,
|
||||
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
|
||||
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
|
||||
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
|
||||
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
|
||||
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := buildAgain(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Refused || r.Revision == 0 {
|
||||
return nil, errors.New(r.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
@@ -264,17 +85,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
||||
a, err := build.DecodeQueueAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
}
|
||||
|
||||
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
||||
// bound for a value written after it that answered says is the builder's answer.
|
||||
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
heard := make(chan json.RawMessage, 1)
|
||||
answered := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
@@ -284,34 +96,36 @@ func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration,
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if answered(c.Value) {
|
||||
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||
select {
|
||||
case heard <- c.Value:
|
||||
case answered <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
||||
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
||||
return e.Value, nil
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||
return e.Value, nil
|
||||
}
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-heard:
|
||||
case v := <-answered:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -352,75 +166,12 @@ func main() {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
||||
a, err := build.DecodeReAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
vs := stdio.State(versions.State)
|
||||
vr := versions.Reader{Keys: vs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := vs.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
ts := stdio.State(versions.TagAsksState)
|
||||
tags := askKept(ts, versions.TagAsksState, versions.TagAsksBucket, "the ask", 25*time.Second, func(v []byte) bool {
|
||||
a, err := versions.DecodeTagAsk(v)
|
||||
return err == nil && !a.Answered.IsZero()
|
||||
})
|
||||
tr := versions.Reader{Keys: ts.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := ts.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
fs := stdio.State(versions.FloorWaitsState)
|
||||
fr := versions.Reader{Keys: fs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := fs.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
ofs := stdio.State(versions.OldFormState)
|
||||
ofr := versions.Reader{Keys: ofs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := ofs.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
bs := stdio.State(versions.BoundsState)
|
||||
br := versions.Reader{Keys: bs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := bs.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr)
|
||||
// The builder's expiring keys (state bounds): what each bounds and when it ends.
|
||||
tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
rows, err := versions.AnswerBounds(br, time.Now())
|
||||
return map[string]any{"bounds": rows}, err
|
||||
}})
|
||||
// What each counted node's engine reads, as the builder keeps it (ADR 0618 §2, ADR 0624 §1): listed, one node asked
|
||||
// again, or one node's key dropped.
|
||||
tools = append(tools, enginereads.Tool(EngineReadsTool, "the builder", enginereads.KeptState(stdio.State("engine-reads")), true, time.Now))
|
||||
// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
|
||||
// states stand, and asking one again by hand (ADR 0615, DLV-M1).
|
||||
tools = append(tools, views.StatusTools(delivery.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), handacts.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// forgeStream is the forge bridge's stream `forge` as the bus names it, where the builder's durable consumer hears a
|
||||
// branch's end (design 51 *Tagged versions*).
|
||||
const forgeStream = "FORGE"
|
||||
|
||||
// forgeBranchDeleted is the forge bridge's kind for a branch's end (delivery.ForgeBranchDeleted in the group's mesh-sdk).
|
||||
const forgeBranchDeleted = "branch-deleted"
|
||||
|
||||
// checkerAsksStream is the checker's family `asks` as the bus names it, and tagWantedSubject its event tag-wanted
|
||||
// (design 51 *Tagged versions*, DLV-C3): on the checker's own stream, so its subject is mesh.stream.<module>.event.*.
|
||||
const (
|
||||
checkerAsksStream = "ASKS"
|
||||
tagWantedSubject = "mesh.stream." + delivery.CheckerModule + ".event.tag-wanted"
|
||||
)
|
||||
|
||||
// forgeEnd is the part of a forge message the builder reads.
|
||||
type forgeEnd struct {
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
// ended says whether the message ends a branch: merged, a pull closed, or branch-deleted.
|
||||
func (f forgeEnd) ended() bool {
|
||||
switch f.Kind {
|
||||
case delivery.ForgeMerged, forgeBranchDeleted:
|
||||
return f.Branch != ""
|
||||
case delivery.ForgePull:
|
||||
return f.Action == delivery.PullClosed && f.Branch != ""
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// branchEnds hears the forge bridge's stream: every message wakes a floor whose bridge_tags was silent, and a branch's
|
||||
// end removes its tagged versions, acknowledged only once all are removed.
|
||||
type branchEnds struct {
|
||||
acts versions.Acts
|
||||
findings *findings
|
||||
tagAsks func() jetstream.KeyValue
|
||||
wake func(ctx context.Context)
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (h *branchEnds) heard(ctx context.Context, m jetstream.Msg) {
|
||||
if h.wake != nil {
|
||||
h.wake(ctx)
|
||||
}
|
||||
var f forgeEnd
|
||||
if err := json.Unmarshal(m.Data(), &f); err != nil || !f.ended() {
|
||||
h.ack(m)
|
||||
return
|
||||
}
|
||||
end, err := h.acts.BranchEnded(ctx, f.Repository, f.Branch)
|
||||
if err != nil {
|
||||
h.logf("the end of %s %s: the versions cannot be read, heard again: %v", f.Repository, f.Branch, err)
|
||||
h.again(m)
|
||||
return
|
||||
}
|
||||
id := f.Repository + "/" + f.Branch
|
||||
if end.Done {
|
||||
h.findings.clear(ctx, versions.RemovalNotDone, health.ScopeModule, id)
|
||||
if kv := h.tagAsks(); kv != nil {
|
||||
if err := kv.Delete(ctx, versions.TagAskKey(f.Repository, f.Branch)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
h.logf("the tag ask of %s %s stays in %s: %v", f.Repository, f.Branch, versions.TagAsksState, err)
|
||||
}
|
||||
}
|
||||
h.ack(m)
|
||||
return
|
||||
}
|
||||
since := h.now()
|
||||
if md, err := m.Metadata(); err == nil {
|
||||
since = md.Timestamp
|
||||
}
|
||||
if h.now().Sub(since) >= versions.RemovalFindingAfter {
|
||||
h.findings.raise(ctx, finding{kind: versions.RemovalNotDone, scope: health.ScopeModule, id: id,
|
||||
summary: end.Finding(f.Repository, f.Branch, since),
|
||||
headline: "A branch's tagged versions are not removed",
|
||||
explanation: "The branch ended, and the registry seats did not say removed for every version tagged for it within 15 minutes.",
|
||||
resolved: "Every seat said removed.",
|
||||
needs: "Look at the seats named: " + strings.Join(end.Seats(), ", ")})
|
||||
}
|
||||
h.logf("the end of %s %s: remove asked of %d tagged versions, heard again in a minute", f.Repository, f.Branch, len(end.Left))
|
||||
h.again(m)
|
||||
}
|
||||
|
||||
func (h *branchEnds) ack(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
h.logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// again leaves the message for the bus to give again in a minute: the retry is the bus's redelivery, not a timer here.
|
||||
func (h *branchEnds) again(m jetstream.Msg) {
|
||||
if err := m.NakWithDelay(versions.RemoveAgainAfter); err != nil {
|
||||
h.logf("leaving %s for the bus to give again: %v", m.Subject(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// tagWanted takes the checker's tag-wanted in: written to tag-asks as builder_tag writes it, answered by its watch.
|
||||
func tagWanted(ctx context.Context, kv jetstream.KeyValue, m jetstream.Msg, by string, now time.Time, logf func(string, ...any)) {
|
||||
if m.Subject() != tagWantedSubject {
|
||||
_ = m.Ack() // the family's other events (the verdicts) are not the builder's
|
||||
return
|
||||
}
|
||||
var t versions.TagWanted
|
||||
err := json.Unmarshal(m.Data(), &t)
|
||||
var key string
|
||||
var value []byte
|
||||
if err == nil {
|
||||
key, value, err = versions.TagWantedAsk(t, by, now)
|
||||
}
|
||||
if err != nil {
|
||||
logf("%s holds what is no tag-wanted, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if kv == nil {
|
||||
logf("tag-wanted of %s %s is heard again: %s is not on the bus yet", t.Repository, t.Branch, versions.TagAsksState)
|
||||
_ = m.NakWithDelay(versions.RemoveAgainAfter)
|
||||
return
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||
logf("tag-wanted of %s %s is heard again: %v", t.Repository, t.Branch, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// tagsVia is build.Builder.Tags over the state tag-asks: the asks of the repository, read when a check's build is
|
||||
// looked at; none while the state is not bound.
|
||||
func tagsVia(kv func() jetstream.KeyValue, logf func(string, ...any)) build.TagOf {
|
||||
return func(ctx context.Context, repository, head string) (string, bool) {
|
||||
store := kv()
|
||||
if store == nil {
|
||||
return "", false
|
||||
}
|
||||
lister, err := store.ListKeysFiltered(ctx, versions.EscapeToken(repository)+".>")
|
||||
if err != nil {
|
||||
logf("the tag asks of %s cannot be read, so its check builds no tagged version: %v", repository, err)
|
||||
return "", false
|
||||
}
|
||||
var asks []versions.TagAsk
|
||||
for key := range lister.Keys() {
|
||||
e, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if t, err := versions.DecodeTagAsk(e.Value()); err == nil {
|
||||
asks = append(asks, t)
|
||||
}
|
||||
}
|
||||
return versions.TagFor(asks, repository, head)
|
||||
}
|
||||
}
|
||||
@@ -123,6 +123,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
|
||||
return catalogue.Decode([]byte(kept.JSON))
|
||||
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
|
||||
}
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/heard"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// findingsBucket is the builder's findings (novox/hq ADR 0564 §2): what it sees wrong now, written at its raise and
|
||||
// deleted at its clear, which the condition keeper raises and clears.
|
||||
var findingsBucket = health.FindingsBucket(delivery.BuilderModule)
|
||||
|
||||
// findings writes the builder's findings into its state, once bound; until then each is said in the log only.
|
||||
type findings struct {
|
||||
kv *process.AwaitedBucket
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
// store stands in for kv in a test.
|
||||
store findingStore
|
||||
}
|
||||
|
||||
// findingStore is the part of the findings bucket written.
|
||||
type findingStore interface {
|
||||
Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
|
||||
}
|
||||
|
||||
// finding is one of the builder's findings: its kind, what it is about, and its words.
|
||||
type finding struct {
|
||||
kind, scope, id string
|
||||
summary, headline, explanation, resolved, needs string
|
||||
severity health.Severity
|
||||
}
|
||||
|
||||
func (f *findings) bucket() findingStore {
|
||||
switch {
|
||||
case f == nil:
|
||||
return nil
|
||||
case f.store != nil:
|
||||
return f.store
|
||||
case f.kv == nil:
|
||||
return nil
|
||||
}
|
||||
if kv := f.kv.KV(); kv != nil {
|
||||
return kv
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildFailed is the finding of a failed trunk build (DLV-B7, design 25), about its module.
|
||||
const BuildFailed = "build-failed"
|
||||
|
||||
// buildEnded raises the module's build-failed for a trunk build that failed and clears it at a later done one; a
|
||||
// candidate's build (its ref named) is the check's to say.
|
||||
func (f *findings) buildEnded(ctx context.Context, e delivery.BuildLogEntry) {
|
||||
if e.Event != delivery.BuildEndedEvent || e.Ref != "" {
|
||||
return
|
||||
}
|
||||
switch e.Outcome {
|
||||
case delivery.BuildFailed:
|
||||
f.raise(ctx, finding{kind: BuildFailed, scope: health.ScopeModule, id: e.Module,
|
||||
summary: "the trunk build of " + e.Module + " at " + e.FP + " (commit " + e.Commit + ") failed",
|
||||
headline: "A module's trunk build failed",
|
||||
explanation: "The builder built " + e.Module + " from the trunk and the build failed, so its new version is not delivered.",
|
||||
resolved: "A later trunk build of " + e.Module + " is done.",
|
||||
needs: "builder_builds_since shows the build; builder_build_again builds it again."})
|
||||
case delivery.BuildDone:
|
||||
f.clear(ctx, BuildFailed, health.ScopeModule, e.Module)
|
||||
}
|
||||
}
|
||||
|
||||
// raise writes fd; it is written again only when its words changed, so the keeper's condition moves only then.
|
||||
func (f *findings) raise(ctx context.Context, fd finding) {
|
||||
key := health.FindingKey(fd.scope, fd.id, fd.kind)
|
||||
kv := f.bucket()
|
||||
if kv == nil {
|
||||
f.logf("finding %s (not written, %s is not on the bus yet): %s", key, findingsBucket, fd.summary)
|
||||
return
|
||||
}
|
||||
if e, err := kv.Get(ctx, key); err == nil {
|
||||
if old, err := health.DecodeFinding(e.Value()); err == nil && old.Summary == fd.summary {
|
||||
return
|
||||
}
|
||||
}
|
||||
at := f.now().UTC()
|
||||
sev := fd.severity
|
||||
if sev == "" {
|
||||
sev = health.Warning
|
||||
}
|
||||
value, err := health.EncodeFinding(health.Finding{Key: key, Kind: fd.kind, Subject: health.Subject{Scope: fd.scope, ID: fd.id},
|
||||
Severity: sev, Summary: fd.summary, Headline: fd.headline, Explanation: fd.explanation, Resolved: fd.resolved, Needs: fd.needs,
|
||||
Evidence: []health.Evidence{{At: at, Said: fd.summary}}, Source: "builder." + fd.kind, Observer: delivery.BuilderModule, At: at})
|
||||
if err == nil {
|
||||
_, err = kv.Put(ctx, key, value)
|
||||
}
|
||||
if err != nil {
|
||||
f.logf("the finding %s cannot be written: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// clear deletes the finding of kind about scope and id; one not there is nothing to clear.
|
||||
func (f *findings) clear(ctx context.Context, kind, scope, id string) {
|
||||
kv := f.bucket()
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
key := health.FindingKey(scope, id, kind)
|
||||
if _, err := kv.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
f.logf("the finding %s cannot be cleared: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// notHeard is the writer of the builder's finding of a family it does not hear (heard.Writer, ADR 0615 §5): written into
|
||||
// its findings once bound, said in the log until then.
|
||||
func (f *findings) notHeard(ctx context.Context) heard.Writer {
|
||||
on := func(do func(findingStore)) {
|
||||
switch {
|
||||
case f.store != nil:
|
||||
do(f.store)
|
||||
case f.kv != nil:
|
||||
f.kv.Ready(func(kv jetstream.KeyValue) { do(kv) })
|
||||
}
|
||||
}
|
||||
return heard.Writer{Observer: delivery.BuilderModule, Logf: f.logf, Now: f.now,
|
||||
Put: func(key string, value []byte) error {
|
||||
on(func(kv findingStore) {
|
||||
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||
f.logf("the finding %s cannot be written: %v", key, err)
|
||||
}
|
||||
})
|
||||
return nil
|
||||
},
|
||||
Delete: func(key string) error {
|
||||
on(func(kv findingStore) {
|
||||
if _, err := kv.Get(ctx, key); err != nil {
|
||||
return // none stands
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil {
|
||||
f.logf("the finding %s cannot be cleared: %v", key, err)
|
||||
}
|
||||
})
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// floorSaid is the floor finding (design 51 *Health and conditions*: a floor source silent): raised for a name waiting on
|
||||
// its floor, about that name, cleared once all three sources answered.
|
||||
func (f *findings) floorSaid(ctx context.Context) func(versions.FloorWait, bool) {
|
||||
return func(w versions.FloorWait, raised bool) {
|
||||
if !raised {
|
||||
f.clear(ctx, versions.FloorSilent, health.ScopeModule, w.Name)
|
||||
return
|
||||
}
|
||||
f.raise(ctx, finding{kind: versions.FloorSilent, scope: health.ScopeModule, id: w.Name, summary: w.Finding(),
|
||||
headline: "A version's floor is not known",
|
||||
explanation: "The builder numbers a name only once its registry seat, the forge bridge's tags and the indexer's requirements all answered; one of them did not.",
|
||||
resolved: "All three answered.",
|
||||
needs: "builder_floor_waits names the silent source; builder_floor_again asks again by hand."})
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -20,7 +19,7 @@ import (
|
||||
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||
// controller had never taken in, and the builder kept them done.
|
||||
|
||||
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
|
||||
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||
// answer}}; a reference is gone while gone says so.
|
||||
type heldVerb struct {
|
||||
mu sync.Mutex
|
||||
@@ -30,7 +29,7 @@ type heldVerb struct {
|
||||
|
||||
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||
t.Helper()
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
References []string `json:"references"`
|
||||
}
|
||||
|
||||
@@ -9,10 +9,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -23,20 +22,10 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
// The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1),
|
||||
// another repository's, by its published version, so the builder is tested against the real pulling code.
|
||||
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -62,7 +51,7 @@ const (
|
||||
)
|
||||
|
||||
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
|
||||
// its build seat's asks on its nodes (runner.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
|
||||
type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
@@ -77,17 +66,14 @@ type fakeBuildModules struct {
|
||||
// closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks
|
||||
// waiting in their streams).
|
||||
closed atomic.Bool
|
||||
pullers []*runner.Puller
|
||||
// hang, while it holds a channel, keeps every build step running until the channel is closed: a step that never ends,
|
||||
// so the builder's step bound runs out (design 51's round-2 case).
|
||||
hang atomic.Pointer[chan struct{}]
|
||||
pullers []*buildstep.Puller
|
||||
}
|
||||
|
||||
// open lets the nodes pull again and wakes their pullers.
|
||||
func (f *fakeBuildModules) open() {
|
||||
f.closed.Store(false)
|
||||
f.mu.Lock()
|
||||
ps := append([]*runner.Puller(nil), f.pullers...)
|
||||
ps := append([]*buildstep.Puller(nil), f.pullers...)
|
||||
f.mu.Unlock()
|
||||
for _, p := range ps {
|
||||
p.Wake()
|
||||
@@ -119,12 +105,6 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any,
|
||||
f.builds = append(f.builds, args)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if h := f.hang.Load(); h != nil && name == "build" {
|
||||
select {
|
||||
case <-*h:
|
||||
case <-f.ctx.Done():
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "fetch":
|
||||
merge := ""
|
||||
@@ -157,26 +137,26 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
language := strings.TrimPrefix(module, "mesh-build-")
|
||||
if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
return kv.Put(f.ctx, key, raw)
|
||||
}
|
||||
call := func(name string) runner.StepFunc {
|
||||
call := func(name string) buildstep.StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
|
||||
}
|
||||
steps := runner.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||
for _, node := range nodes {
|
||||
p := &runner.Puller{Source: &runner.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||
Capacity: func() int {
|
||||
if f.closed.Load() {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }}
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||
f.mu.Lock()
|
||||
f.pullers = append(f.pullers, p)
|
||||
f.mu.Unlock()
|
||||
@@ -210,11 +190,7 @@ type bed struct {
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
instances jetstream.KeyValue
|
||||
tagAsks jetstream.KeyValue
|
||||
tagBounds jetstream.KeyValue
|
||||
packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family
|
||||
out *saidLog // the builder's process log, every instance's
|
||||
stop func() // stops the builder started last
|
||||
stop func() // stops the builder started last
|
||||
fake *fakeBuildModules
|
||||
srvURL string
|
||||
dir string
|
||||
@@ -289,29 +265,8 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
// The indexer and the assigner as the owners of the states the builder follows: their read tools and views families
|
||||
// (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer.
|
||||
readers := []string{"n1_" + delivery.BuilderModule, "n2_" + delivery.BuilderModule}
|
||||
for _, st := range []string{modulesState, manifestsState, candidatesState} {
|
||||
viewstest.Serve(t, ctx, js, ctl, delivery.IndexerModule, st, delivery.IndexerModule+".indexer_read_"+st, indexerViews, readers...)
|
||||
}
|
||||
b.packages = viewstest.Serve(t, ctx, js, ctl, delivery.IndexerModule, delivery.PackagesState, delivery.IndexerModule+".indexer_read_packages", indexerViews, readers...)
|
||||
viewstest.Serve(t, ctx, js, ctl, delivery.AssignerModule, delivery.PlacementState, delivery.AssignerModule+".assigner_read_placement", assignerViews, readers...)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
// The tag asks and their bounds (design 51 *Tagged versions*): the bounds declared key-ttl, so an ended bound leaves the
|
||||
// bus's marker a watch hears.
|
||||
b.tagAsks = mk(versions.TagAsksBucket, 1)
|
||||
if b.tagBounds, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: versions.TagAskBoundsBucket, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.out = &saidLog{}
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
|
||||
logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"},
|
||||
AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||
b.fake.closed.Store(closed)
|
||||
b.fake.serve(t, "mesh-build-go", "n1", "n2")
|
||||
@@ -324,29 +279,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
return b
|
||||
}
|
||||
|
||||
// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user.
|
||||
func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry {
|
||||
t.Helper()
|
||||
conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []delivery.BuildLogEntry
|
||||
for _, le := range entries {
|
||||
e, err := delivery.DecodeBuildLogEntry(le.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
||||
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
||||
t.Helper()
|
||||
@@ -365,24 +297,15 @@ func (b *bed) report(t *testing.T, module, node, language, version, recipe strin
|
||||
|
||||
// module writes modules.<name> as the indexer does for a module with a new-form fingerprint, and its manifest by digest.
|
||||
func (b *bed) module(t *testing.T, name, fixture, path string) string {
|
||||
return b.moduleWith(t, name, fixture, path, bothToolchains)
|
||||
return b.moduleWith(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})
|
||||
}
|
||||
|
||||
// bothToolchains are the toolchains the bed's nodes report, as a module's fingerprint names them.
|
||||
var bothToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
|
||||
|
||||
// moduleWith is module with the toolchains its fingerprint was computed with.
|
||||
func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed) string {
|
||||
return b.moduleCommit(t, name, fixture, path, toolchains, commit)
|
||||
}
|
||||
|
||||
func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at string) string {
|
||||
t.Helper()
|
||||
return b.moduleFP(t, name, fixture, path, toolchains, at, fp, 0)
|
||||
}
|
||||
|
||||
// moduleFP is moduleCommit with the new-form fingerprint fp2 and the source revision src of its merge (0: none).
|
||||
func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at, fp2 string, src uint64) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
@@ -400,8 +323,8 @@ func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[
|
||||
if _, err := manifests.Put(b.ctx, digest, mv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp2,
|
||||
Manifest: digest, Path: path, Toolchains: toolchains, Src: src})
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp,
|
||||
Manifest: digest, Path: path, Toolchains: toolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -422,7 +345,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -432,58 +355,16 @@ func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
}
|
||||
runCtx, stop := context.WithCancel(b.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return e[k] }, b.out) }()
|
||||
go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
|
||||
var once sync.Once
|
||||
b.stop = func() { once.Do(func() { stop(); <-done }) }
|
||||
t.Cleanup(b.stop)
|
||||
}
|
||||
|
||||
// saidLog is the builder's process log as its instances write it.
|
||||
type saidLog struct {
|
||||
mu sync.Mutex
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (l *saidLog) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.buf.Write(p)
|
||||
}
|
||||
|
||||
// awaitSaid reads the process log until it holds want, within 20 s, and answers the line.
|
||||
func (b *bed) awaitSaid(t *testing.T, want string) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for {
|
||||
b.out.mu.Lock()
|
||||
all := b.out.buf.String()
|
||||
b.out.mu.Unlock()
|
||||
for _, line := range strings.Split(all, "\n") {
|
||||
if strings.Contains(line, want) {
|
||||
return line
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the builder never said %q; it said:\n%s", want, all)
|
||||
}
|
||||
select {
|
||||
case <-b.ctx.Done():
|
||||
t.Fatal(b.ctx.Err())
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// await reads the build key until pred holds of it, within 20 s.
|
||||
func (b *bed) await(t *testing.T, module string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild {
|
||||
t.Helper()
|
||||
return b.awaitFP(t, module, fp, pred)
|
||||
}
|
||||
|
||||
// awaitFP is await of the build key of fingerprint at.
|
||||
func (b *bed) awaitFP(t *testing.T, module, at string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild {
|
||||
t.Helper()
|
||||
key, err := delivery.BuilderBuildKey(module, at)
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -510,31 +391,9 @@ func isState(state string) func(delivery.BuilderBuild) bool {
|
||||
// the digests of both bundles, both build modules' toolchains, after a fetch and a build on one node (ADR 0478 clause 4).
|
||||
func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
b := startBed(t)
|
||||
// DLV-B10 (design 51, round 2): the builder's versions, findings and the floor's three sources are on the bus; the seat
|
||||
// answers 0.1.4, so netcheck's bundles are numbered from 0.1.5. Two merges close together, the first replaced before
|
||||
// its claim: the second is numbered at once, and the first never.
|
||||
reg := b.registry(t, "n1", "0.1.4")
|
||||
const netcheck = "catalogue-a67f0196-netcheck.module.json"
|
||||
first := "src2:" + strings.Repeat("1", 64)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, first, 1)
|
||||
digest := b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, fp, 2)
|
||||
// DLV-B6: a tag ask kept and never numbered (no check builds its head) is narrated once its bound ends on the bus, with
|
||||
// what it waits on, and its bound is written again; no timer runs in the builder.
|
||||
key, ask, err := versions.NewTagAsk("novox/mesh-sdk", "feat/x", "abc", "a group member's check", "me@shanks", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.tagAsks.Put(b.ctx, key, ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.start(t, "n1", map[string]string{"MESH_BUILDER_TAG_ASK_BOUND": "1s", "MESH_BUILDER_STEP_BOUND": "1s"})
|
||||
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "netcheck", isState(delivery.BuildDone))
|
||||
for _, a := range []string{"tools-go", "tools-typescript"} {
|
||||
es := reg.of(t, "netcheck/"+a)
|
||||
if len(es) != 1 || es[0].Version != "0.1.5" || es[0].Fingerprint != fp || es[0].State != versions.Numbered || es[0].Digest != digestOf(a) {
|
||||
t.Errorf("versions of netcheck/%s = %+v, want the second merge numbered 0.1.5 at once with its digest, the first never", a, es)
|
||||
}
|
||||
}
|
||||
if v.By != "n1.mesh-delivery-builder" || v.Manifest != digest || len(v.Artifacts) != 2 {
|
||||
t.Fatalf("build = %+v", v)
|
||||
}
|
||||
@@ -546,193 +405,39 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
|
||||
t.Errorf("toolchains = %+v", v.Toolchains)
|
||||
}
|
||||
// ADR 0618 §1: the done key names the reference forms it is written in.
|
||||
if len(v.Needs) != 1 || v.Needs[0] != "form:artifact-store" {
|
||||
t.Errorf("needs = %v, want the first form", v.Needs)
|
||||
}
|
||||
// 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a
|
||||
// reader with the log's read reads them, the end naming the manifest and what was made.
|
||||
// DLV-B10: between them the build log holds the build's narrations (build.said), each version numbered, and why.
|
||||
var ended []delivery.BuildLogEntry
|
||||
for _, e := range b.logged(t, "netcheck") {
|
||||
if e.Event != delivery.BuildSaidEvent {
|
||||
ended = append(ended, e)
|
||||
} else if e.Why != versions.SaidNumbered || !strings.Contains(e.Line, "numbered 0.1.5") || e.FP != fp {
|
||||
t.Errorf("said = %+v, want netcheck's versions numbered", e)
|
||||
}
|
||||
}
|
||||
if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent ||
|
||||
ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" {
|
||||
t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended)
|
||||
}
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
|
||||
if got := b.fake.count(tool); got != want {
|
||||
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
|
||||
}
|
||||
}
|
||||
said := b.awaitSaid(t, "the tag ask of novox/mesh-sdk feat/x at abc (feat-x) is not answered after")
|
||||
if !strings.Contains(said, "the check's build of abc is not claimed yet") {
|
||||
t.Errorf("the narration = %q, want what the ask waits on", said)
|
||||
}
|
||||
// And in the build log, under the builder's own key (the ruling of 13:17: build.said).
|
||||
if !slices.ContainsFunc(b.logged(t, delivery.BuilderModule), func(e delivery.BuildLogEntry) bool {
|
||||
return e.Event == delivery.BuildSaidEvent && e.Why == versions.SaidUnanswered && e.FP == key && e.Commit == "abc" &&
|
||||
strings.Contains(e.Line, "the check's build of abc is not claimed yet")
|
||||
}) {
|
||||
t.Errorf("the build log of %s = %+v, want the unanswered tag ask said", delivery.BuilderModule, b.logged(t, delivery.BuilderModule))
|
||||
}
|
||||
if e, err := b.tagAsks.Get(b.ctx, key); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if a, _ := versions.DecodeTagAsk(e.Value()); a.Answered.IsZero() || !a.Unanswered() {
|
||||
t.Errorf("the tag ask = %+v, want kept and answered, no version yet", a)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
if _, err := b.tagBounds.Get(b.ctx, key); err == nil {
|
||||
break
|
||||
} else if time.Now().After(deadline) {
|
||||
t.Fatalf("the bound is not written again after its narration: %v", err)
|
||||
}
|
||||
}
|
||||
// Design 32, ADR 0615 §1: a package's name the indexer writes later reaches the builder by its packages' views family
|
||||
// (never their bucket), and looks at the module again; the done bundles are kept, not numbered twice.
|
||||
if _, err := b.packages.Put(b.ctx, "netcheck.0.1", []byte(`{"v":1,"name":"netcheck","package":"netcheck/lib","series":"0.1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.awaitSaid(t, "names a package of netcheck")
|
||||
if got := reg.numberedOf("netcheck/tools-go"); len(got) != 1 || got[0] != "0.1.5" {
|
||||
t.Errorf("version.numbered of netcheck/tools-go = %v, want 0.1.5 once", got)
|
||||
}
|
||||
// DLV-B10: builder_publish_again after a published takes a new number, and says it numbered.
|
||||
for _, a := range []string{"tools-go", "tools-typescript"} {
|
||||
reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a))
|
||||
}
|
||||
reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Published })
|
||||
key, again, err := build.NewPublishAgain("netcheck/tools-go", fp, "the holder lost it", "me@shanks", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := reg.answered(t, key, again); a.Refused || a.Do != versions.AgainRenumber || a.Version != "0.1.6" {
|
||||
t.Errorf("builder_publish_again = %+v, want a new number, 0.1.6", a)
|
||||
}
|
||||
// Running-testing note (review round 1): two claims never get one number on the bus's own compare-and-set, as
|
||||
// the builder's state versions keeps them: eight claims of one name at once, eight versions.
|
||||
store := versions.Versions{Store: awaitedVersions{process.AwaitBucket(b.ctx, b.js, nil, versions.Bucket, "", t.Logf)}}
|
||||
numbers := make(chan string, 8)
|
||||
var claims sync.WaitGroup
|
||||
for i := range 8 {
|
||||
claims.Go(func() {
|
||||
d, _, err := store.Number(b.ctx, versions.Claim{Name: "netcheck/race", Kind: versions.KindBundle, Fingerprint: fmt.Sprintf("f%d", i)})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
numbers <- d.Version
|
||||
})
|
||||
}
|
||||
claims.Wait()
|
||||
close(numbers)
|
||||
got := map[string]bool{}
|
||||
for v := range numbers {
|
||||
got[v] = true
|
||||
}
|
||||
if len(got) != 8 {
|
||||
t.Errorf("eight claims of one name at once were numbered %v, want eight versions", slices.Sorted(maps.Keys(got)))
|
||||
}
|
||||
if got := reg.numberedOf("netcheck/tools-go"); len(got) != 2 || got[1] != "0.1.6" {
|
||||
t.Errorf("version.numbered of netcheck/tools-go = %v, want 0.1.6 said after 0.1.5", got)
|
||||
}
|
||||
// DLV-B10: a build module's bound running out fails the trunk build, which is the builder's finding build-failed; the
|
||||
// fingerprint built again keeps its number, said with why, and its done build clears the finding.
|
||||
release := make(chan struct{})
|
||||
b.fake.hang.Store(&release)
|
||||
third := "src2:" + strings.Repeat("4", 64)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3)
|
||||
if f := b.awaitFP(t, "netcheck", third, isState(delivery.BuildFailed)); !strings.Contains(f.Error, "did not end within") {
|
||||
t.Errorf("the failed build = %+v, want its bound run out", f)
|
||||
}
|
||||
failed := health.FindingKey(health.ScopeModule, "netcheck", BuildFailed)
|
||||
reg.awaitFinding(t, failed, true)
|
||||
b.fake.hang.Store(nil)
|
||||
close(release)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) // the indexer writes it again
|
||||
b.awaitFP(t, "netcheck", third, isState(delivery.BuildDone))
|
||||
b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it")
|
||||
if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e delivery.BuildLogEntry) bool {
|
||||
return e.Event == delivery.BuildSaidEvent && e.Why == versions.SaidKept && e.FP == third && strings.Contains(e.Line, "kept: fingerprint "+third)
|
||||
}) {
|
||||
t.Errorf("the build log of netcheck says no version kept at %s", third)
|
||||
}
|
||||
reg.awaitFinding(t, failed, false)
|
||||
// DLV-B4: the seats' refused and removed, heard on the builder's durable consumer, move a numbered version: a retention
|
||||
// removal turns the entry removed with its reason, a refusal refused with the holder's reason and the bytes it refused.
|
||||
reg.outcome(t, ociartifactregistry.SubjectRemoved, versions.Outcome{Name: "netcheck/tools-typescript", Version: "0.1.5", Kind: "bundle", Holder: "distribution", Reason: "retention"})
|
||||
reg.outcome(t, ociartifactregistry.SubjectRefused, versions.Outcome{Name: "netcheck/tools-go", Version: "0.1.6", Digest: digestOf("other"), Kind: "bundle", Holder: "distribution", Reason: "the cap"})
|
||||
reg.awaitOf(t, "netcheck/tools-typescript", func(es []versions.Entry) bool {
|
||||
return slices.ContainsFunc(es, func(e versions.Entry) bool {
|
||||
return e.Version == "0.1.5" && e.State == versions.Removed && e.Reason == "retention"
|
||||
})
|
||||
})
|
||||
reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool {
|
||||
return slices.ContainsFunc(es, func(e versions.Entry) bool {
|
||||
return e.Version == "0.1.6" && e.State == versions.Refused && e.Reason == "the cap" && e.RefusedDigest == digestOf("other")
|
||||
})
|
||||
})
|
||||
// DLV-B6: a branch's end (the forge bridge's merged) asks the seat to remove each version tagged for it, the reason
|
||||
// tag-gone, and the seat's removed turns it removed.
|
||||
tagged := versions.Entry{Name: "mesh-sdk/go", Kind: versions.KindPackage, Series: "0.1", Version: "0.1.15-feat-x.1", Tag: "feat-x", Module: "mesh-sdk",
|
||||
Fingerprint: "src2:" + strings.Repeat("5", 64), Repository: "novox/mesh-sdk", Seat: gopackageregistry.Seat, State: versions.Published, At: time.Now().UTC()}
|
||||
raw, _ := json.Marshal(tagged)
|
||||
if _, err := reg.versions.Put(b.ctx, versions.Key(tagged.Name, tagged.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(delivery.ForgeBridgeModule, "forge"),
|
||||
[]byte(`{"kind":"`+delivery.ForgeMerged+`","repository":"novox/mesh-sdk","branch":"feat/x"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
reg.mu.Lock()
|
||||
asked := slices.ContainsFunc(reg.removes, func(a map[string]any) bool {
|
||||
return a["name"] == tagged.Name && a["version"] == tagged.Version && a["reason"] == "tag-gone"
|
||||
})
|
||||
reg.mu.Unlock()
|
||||
if asked {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the seat's remove of %s@%s was not asked at the branch's end: %v", tagged.Name, tagged.Version, reg.removes)
|
||||
}
|
||||
}
|
||||
reg.outcome(t, gopackageregistry.SubjectRemoved, versions.Outcome{Name: tagged.Name, Version: tagged.Version, Kind: versions.KindPackage, Holder: "gitea", Reason: "tag-gone"})
|
||||
reg.awaitOf(t, tagged.Name, func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Removed })
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "."
|
||||
// is given to the build tools as it is (1615/AC4). Two builders on two nodes start together on the archive's key: the
|
||||
// second takes over the key the first holds building (ADR 0478 clause 6), and the key ends done, written once (1607/AC7).
|
||||
// One table: three tests were merged, to pay for the enrolment's and to bring mesh-delivery's count down (DLV-B8).
|
||||
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, path string
|
||||
two bool
|
||||
}{{"no toolchain", "modules/ca-trust", false}, {"root", delivery.RootPath, false}, {"two instances together", "modules/ca-trust", true}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", c.path, commit)
|
||||
b.start(t, "n1", nil)
|
||||
if c.two {
|
||||
b.start(t, "n2", nil)
|
||||
}
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
if c.two && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) {
|
||||
t.Errorf("build = %+v, want the archive's digest", v)
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); c.two && n < 1 { // each instance that built, on one node
|
||||
t.Errorf("build_go_build was called %d times, want one for each instance that built", n)
|
||||
}
|
||||
})
|
||||
// TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey: two builders on two nodes start together on one module key of an
|
||||
// archive only (no toolchain named, as the indexer writes it); the second takes over the key the first holds building
|
||||
// (ADR 0478 clause 6), and the key ends done, written once.
|
||||
func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.start(t, "n2", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files") {
|
||||
t.Fatalf("build = %+v, want the archive's digest", v)
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); n < 1 { // each instance that built, on one node
|
||||
t.Errorf("build_go_build was called %d times, want one for each instance that built", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4).
|
||||
func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -787,6 +492,14 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
|
||||
}
|
||||
}
|
||||
|
||||
// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
|
||||
func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", delivery.RootPath, commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
}
|
||||
|
||||
func mustKey(t *testing.T, module string) string {
|
||||
k, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
@@ -1040,12 +753,7 @@ func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) {
|
||||
}
|
||||
defer sub.Close()
|
||||
var calls atomic.Int64
|
||||
// A build module's tool, never the owners' read tools the builder's first views ask (DLV-M1).
|
||||
s, err := sub.Subscribe("mesh.mod.*.call.>", func(m *nats.Msg) {
|
||||
if strings.HasPrefix(m.Subject, "mesh.mod.mesh-build-") {
|
||||
calls.Add(1)
|
||||
}
|
||||
})
|
||||
s, err := sub.Subscribe("mesh.mod.*.call.>", func(*nats.Msg) { calls.Add(1) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1164,184 +872,3 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
t.Errorf("done = %+v", done)
|
||||
}
|
||||
}
|
||||
|
||||
// registry is the part of the bed design 51's builder cases need: the builder's states versions, findings, floor-waits and
|
||||
// reasks, EVENTS with the seat oci-artifact-registry's outcomes and the builder's durable consumer of it, the builder's
|
||||
// own events kept, and the floor's three sources served on the bus as their holders answer through the tool runner
|
||||
// ({result}): the seat's versions, the forge bridge's bridge_tags and the indexer's indexer_requirements.
|
||||
type registry struct {
|
||||
b *bed
|
||||
versions, findings, reasks jetstream.KeyValue
|
||||
mu sync.Mutex
|
||||
numbered []delivery.VersionNumbered
|
||||
removes []map[string]any
|
||||
}
|
||||
|
||||
func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *registry {
|
||||
t.Helper()
|
||||
r := ®istry{b: b}
|
||||
mk := func(name string) jetstream.KeyValue {
|
||||
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: name, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
r.versions, r.findings, r.reasks = mk(versions.Bucket), mk(findingsBucket), mk(build.ReAsksBucket)
|
||||
mk(versions.FloorWaitsBucket)
|
||||
// EVENTS with two seats' outcomes, and the forge bridge's stream (a branch's end): each with the builder's durable consumer.
|
||||
for stream, subjects := range map[string][]string{
|
||||
eventsStream: {delivery.ArtifactsGoneSubject, ociartifactregistry.SubjectPublished, ociartifactregistry.SubjectRefused, ociartifactregistry.SubjectRemoved,
|
||||
gopackageregistry.SubjectPublished, gopackageregistry.SubjectRefused, gopackageregistry.SubjectRemoved},
|
||||
forgeStream: {process.ModuleStreamSubject(delivery.ForgeBridgeModule, ">")},
|
||||
} {
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: stream, Subjects: subjects}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream, jetstream.ConsumerConfig{Durable: node + "_" + delivery.BuilderModule,
|
||||
AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The Go seat's remove, as its holder answers it; each call is kept.
|
||||
rm, err := b.js.Conn().Subscribe("mesh.seat."+gopackageregistry.Seat+".call.remove.>", func(m *nats.Msg) {
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
r.mu.Lock()
|
||||
r.removes = append(r.removes, args)
|
||||
r.mu.Unlock()
|
||||
_ = m.Respond([]byte(`{"result":{},"node":"ctl"}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = rm.Unsubscribe() })
|
||||
own := "mesh.mod." + delivery.BuilderModule + ".event.>"
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "BUILDER_EVENTS", Subjects: []string{own}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nc := b.js.Conn()
|
||||
// As a build module hears it: on the contract's subject, read by the contract's decoder (BG2's late publish).
|
||||
sub, err := nc.Subscribe(delivery.VersionNumberedSubject, func(m *nats.Msg) {
|
||||
if nb, err := delivery.DecodeVersionNumbered(m.Data); err == nil {
|
||||
r.mu.Lock()
|
||||
r.numbered = append(r.numbered, nb)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
var listed []map[string]string
|
||||
for _, v := range seatVersions {
|
||||
listed = append(listed, map[string]string{"version": v})
|
||||
}
|
||||
for subject, answer := range map[string]any{
|
||||
"mesh.seat." + ociartifactregistry.Seat + ".call.versions.>": map[string]any{"versions": listed},
|
||||
"mesh.mod.mesh-delivery-forge-bridge.call.bridge_tags.>": map[string]any{"tags": []string{}},
|
||||
"mesh.mod." + delivery.IndexerModule + ".call.indexer_requirements.>": map[string]any{"requirements": []string{}},
|
||||
} {
|
||||
reply, _ := json.Marshal(map[string]any{"result": answer, "node": "ctl"})
|
||||
s, err := nc.Subscribe(subject, func(m *nats.Msg) { _ = m.Respond(reply) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Unsubscribe() })
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// of is every entry versions holds of name.
|
||||
func (r *registry) of(t *testing.T, name string) []versions.Entry {
|
||||
t.Helper()
|
||||
l, err := r.versions.ListKeysFiltered(r.b.ctx, versions.NameFilter(name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = l.Stop() }()
|
||||
var out []versions.Entry
|
||||
for k := range l.Keys() {
|
||||
e, err := r.versions.Get(r.b.ctx, k)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v versions.Entry
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// awaitOf reads name's entries until pred holds of them, within 20 s.
|
||||
func (r *registry) awaitOf(t *testing.T, name string, pred func([]versions.Entry) bool) []versions.Entry {
|
||||
t.Helper()
|
||||
var es []versions.Entry
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if es = r.of(t, name); pred(es) {
|
||||
return es
|
||||
}
|
||||
}
|
||||
t.Fatalf("versions of %s = %+v within 20 s", name, es)
|
||||
return es
|
||||
}
|
||||
|
||||
// published says the artifact seat's published for name@version, as its holder does.
|
||||
func (r *registry) published(t *testing.T, name, version, digest string) {
|
||||
t.Helper()
|
||||
r.outcome(t, ociartifactregistry.SubjectPublished, versions.Outcome{Name: name, Version: version, Digest: digest, Kind: "bundle", Holder: "distribution"})
|
||||
}
|
||||
|
||||
// outcome says a seat's outcome on subject, as its holder does.
|
||||
func (r *registry) outcome(t *testing.T, subject string, o versions.Outcome) {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(o)
|
||||
msg := nats.NewMsg(subject)
|
||||
msg.Header.Set("x-event-id", subject+"-"+o.Name+"-"+o.Version)
|
||||
msg.Data = data
|
||||
if _, err := r.b.js.PublishMsg(r.b.ctx, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// answered writes a reasks value and reads it until the builder answered it, within 20 s.
|
||||
func (r *registry) answered(t *testing.T, key string, value []byte) build.ReAsk {
|
||||
t.Helper()
|
||||
if _, err := r.reasks.Put(r.b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var a build.ReAsk
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := r.reasks.Get(r.b.ctx, key); err == nil && json.Unmarshal(e.Value(), &a) == nil && !a.Answered.IsZero() {
|
||||
return a
|
||||
}
|
||||
}
|
||||
t.Fatalf("reasks.%s = %+v, not answered within 20 s", key, a)
|
||||
return a
|
||||
}
|
||||
|
||||
// awaitFinding reads the builder's finding key until it is there (raised) or gone, within 20 s.
|
||||
func (r *registry) awaitFinding(t *testing.T, key string, raised bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if _, err := r.findings.Get(r.b.ctx, key); (err == nil) == raised {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("the finding %s raised = %v within 20 s, want %v", key, !raised, raised)
|
||||
}
|
||||
|
||||
// numberedOf are the version.numbered the builder said for name, in order.
|
||||
func (r *registry) numberedOf(name string) []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []string
|
||||
for _, nb := range r.numbered {
|
||||
if nb.Name == name {
|
||||
out = append(out, nb.Version)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -5,8 +5,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -19,18 +24,15 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/forms"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -49,123 +51,118 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
|
||||
cred, err := process.ReadBrokerAccount(getenv)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, js, err := cred.Dial()
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The indexer's modules, manifests, candidates and packages and the assigner's placement are their owners' states, read by the
|
||||
// first view of their read tools and then their views families, never their buckets (ADR 0615 §1, DLV-M1). An owner that
|
||||
// is down never holds the start: each is read on a goroutine once its first view is answered, and until then the
|
||||
// unanswered first view is the builder's finding.
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
reader := &views.ProcessReader{Module: delivery.BuilderModule, Call: calls.Call, Logf: logger.Printf}
|
||||
findingsKV := process.AwaitBucket(ctx, js, conn, findingsBucket, "the builder's findings, a first view unanswered past its bound among them, are said in its log only", logger.Printf)
|
||||
reader.Findings = findingsKV.KV
|
||||
modules, err := reader.Follow(delivery.IndexerModule, modulesState, delivery.IndexerModule+".indexer_read_modules", indexerViews)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := reader.Follow(delivery.IndexerModule, manifestsState, delivery.IndexerModule+".indexer_read_manifests", indexerViews)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684).
|
||||
candidates, err := reader.Follow(delivery.IndexerModule, candidatesState, delivery.IndexerModule+".indexer_read_candidates", indexerViews)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The indexer's packages, whose names a package is numbered under (design 32).
|
||||
packages, err := reader.Follow(delivery.IndexerModule, delivery.PackagesState, delivery.IndexerModule+".indexer_read_packages", indexerViews)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
placement, err := reader.Follow(delivery.AssignerModule, delivery.PlacementState, delivery.AssignerModule+".assigner_read_placement", assignerViews)
|
||||
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
||||
}
|
||||
}
|
||||
// An unanswered tag ask is narrated every TagAskBound (design 51: five minutes); a test gives a shorter one.
|
||||
tagAskBound := versions.TagAskBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_TAG_ASK_BOUND")); v != "" {
|
||||
if tagAskBound, err = time.ParseDuration(v); err != nil {
|
||||
return fmt.Errorf("MESH_BUILDER_TAG_ASK_BOUND: %w", err)
|
||||
}
|
||||
}
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
// state hearing, which builder_stuck says (dead_letters).
|
||||
dead := &buildrun.DeadLetters{}
|
||||
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
|
||||
// is bound when the bus says it was created, and the latest hearing is written then.
|
||||
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
|
||||
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
|
||||
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
var buildModules []string
|
||||
for _, l := range delivery.AskLanguages {
|
||||
if m, err := delivery.BuildModuleOf(l); err == nil {
|
||||
buildModules = append(buildModules, m)
|
||||
}
|
||||
}
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection.
|
||||
// The builder numbers each trunk build at its claim (novox/hq design 51 *The versions*): its state versions, bound once
|
||||
// the bus holds it; the floor's three sources asked as its own account; each new number said as version.numbered.
|
||||
versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf)
|
||||
found := &findings{kv: findingsKV, logf: logger.Printf, now: time.Now}
|
||||
floorWaits := &versions.FloorWaits{Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))}
|
||||
// The names waiting on their floor are kept in the state floor-waits (DLV-B4): a restarted builder takes them in, so
|
||||
// their silent sources' events still wake them; until the state is bound they are kept here only.
|
||||
process.AwaitBucket(ctx, js, conn, versions.FloorWaitsBucket, "the names waiting on their floor are kept in memory only, and builder_floor_waits shows none", logger.Printf).
|
||||
Ready(func(kv jetstream.KeyValue) {
|
||||
floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf))
|
||||
})
|
||||
// A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32).
|
||||
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits,
|
||||
Names: versions.NamesIn(indexerPackages{packages}),
|
||||
Said: func(nb delivery.VersionNumbered) {
|
||||
if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}}
|
||||
// A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder
|
||||
// answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read.
|
||||
pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf)
|
||||
pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf}
|
||||
numberer.Publishing = pubs.arm
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
@@ -174,55 +171,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
}
|
||||
tagAsksKV := process.AwaitBucket(ctx, js, conn, versions.TagAsksBucket, "builder_tag and builder_untag are not answered", logger.Printf)
|
||||
// A check's build of a head a kept tag ask names numbers its packages tagged (design 51 *Tagged versions*), and the ask
|
||||
// is marked numbered with its versions.
|
||||
b.Tags = tagsVia(tagAsksKV.KV, logger.Printf)
|
||||
b.Numbers = build.VersionNumbers{N: numberer, TagNumbered: tagNumbered(tagAsksKV.KV, logger.Printf, time.Now)}
|
||||
// An ask not answered within five minutes is narrated with what it waits on: its bound's end on the bus says when.
|
||||
boundsKV := process.AwaitBucket(ctx, js, conn, versions.TagAskBoundsBucket, "an unanswered tag ask is not narrated", logger.Printf)
|
||||
bounds := &tagBounds{asks: tagAsksKV.KV, bounds: boundsKV.KV, bound: tagAskBound, waits: floorWaits.Waiting, logf: logger.Printf, now: time.Now,
|
||||
log: func(e delivery.BuildLogEntry) { b.Log(e) }}
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
|
||||
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
|
||||
acts := versions.Acts{Numberer: numberer, Done: doneVia(b), Reconsider: b.Reconsider, Remove: removeVia(calls.Call),
|
||||
BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) {
|
||||
r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound)
|
||||
if r.Refused != "" {
|
||||
return r.Refused, true
|
||||
}
|
||||
if r.Failed != "" {
|
||||
return "built again, and it failed before its asks were put: " + r.Failed, false
|
||||
}
|
||||
return fmt.Sprintf("built again as revision %d", r.Revision), false
|
||||
}}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 10)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) {
|
||||
// A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*).
|
||||
(&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "")
|
||||
b.TrunkMoved(ctx)
|
||||
}, errs: errs, logf: logger.Printf}
|
||||
errs := make(chan error, 6)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||
b.Log = func(e delivery.BuildLogEntry) {
|
||||
if e.Node == "" { // a narration of no one build (a tag ask, the old form's window) is this instance's
|
||||
e.Node = cred.Node
|
||||
}
|
||||
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
// A failed trunk build is the builder's finding build-failed, cleared by a later done one (DLV-B7).
|
||||
found.buildEnded(context.WithoutCancel(ctx), e)
|
||||
}
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
@@ -247,223 +204,48 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// placement is settled first, so the toolchains settle through the gate already knowing it.
|
||||
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||
watch.put = gate.Toolchain
|
||||
// What each counted node's engine reads (ADR 0618 §2, ADR 0624 §1): asked through the node-engine seat's verb as the
|
||||
// builder's own account, for every node the placement names, and kept in the state engine-reads once it is bound.
|
||||
reads := newEngineReads(calls.CallOn, gate, logger.Printf)
|
||||
// The old reference form's window over them (DLV-B9), kept in the state old-form once it is bound; builder_old_form
|
||||
// reads it and drops a retired node by hand.
|
||||
reads.ctx = context.WithoutCancel(ctx)
|
||||
reads.window = &oldForm{forms: forms.Current, findings: found, logf: logger.Printf, now: time.Now, log: func(e delivery.BuildLogEntry) { b.Log(e) },
|
||||
bounds: pubBoundsKV.KV,
|
||||
said: func(body versions.OldFormClosedBody) {
|
||||
if err := emitEvent(context.WithoutCancel(ctx), js, cred, versions.OldFormClosedEvent, "old-form-closed-"+body.At.Format(time.RFC3339Nano), body); err != nil {
|
||||
logger.Printf("saying %s: %v", versions.OldFormClosedEvent, err)
|
||||
}
|
||||
}}
|
||||
acts.DropOldForm = func(ctx context.Context, node, by, why string, at time.Time) (string, error) {
|
||||
answer, err := reads.window.drop(ctx, node, by, why, at)
|
||||
if err == nil {
|
||||
reads.stepWindow()
|
||||
}
|
||||
return answer, err
|
||||
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
process.AwaitBucket(ctx, js, conn, versions.OldFormBucket, "the old form's window is kept in memory only, and builder_old_form shows none", logger.Printf).
|
||||
Ready(func(kv jetstream.KeyValue) {
|
||||
reads.window.bind(context.WithoutCancel(ctx), kv)
|
||||
reads.stepWindow()
|
||||
})
|
||||
gate.OnPlaced = reads.placed
|
||||
b.Forms = reads.built
|
||||
// Until the placement's first view is answered the gate passes every report on, as before the placement is settled.
|
||||
placementErrs := make(chan error, 1)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-placement.Follower().Ready():
|
||||
}
|
||||
pe, err := buildrun.FollowPlacement(ctx, reader, gate, b.NodesChanged, logger.Printf)
|
||||
if err != nil {
|
||||
placementErrs <- err
|
||||
return
|
||||
}
|
||||
placementErrs <- <-pe
|
||||
}()
|
||||
if err := watch.start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the builder's durable consumer on it, are
|
||||
// made by the controller. Where the stream is not there yet the builder says so and is started again with the build
|
||||
// module.
|
||||
// Each consumer that is not there yet is the finding family-not-heard naming its stream until it binds at a later
|
||||
// start (heard.Writer; ADR 0615 §5); one there that cannot be heard stops the builder, as before.
|
||||
notHeard := found.notHeard(ctx)
|
||||
var stopped error
|
||||
hear := func(stream, consequence string, each func(jetstream.Msg)) {
|
||||
_ = notHeard.Bind(stream, consequence, func() error {
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s does not exist yet: %w", cred.Node, cred.Module, stream, err)
|
||||
}
|
||||
cc, err := process.HearEvents(cons, each, logger.Printf)
|
||||
if err != nil {
|
||||
stopped = cmp.Or(stopped, err)
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
|
||||
hear(toolchainStreams[module], module+"'s toolchain reports are not heard until the builder starts again", func(m jetstream.Msg) { watch.Reported(m) })
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
// The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the
|
||||
// builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says
|
||||
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||
// so; it still asks held at its start and at every module value.
|
||||
// The same consumer carries what wakes a floor (design 51 *The floor*): the indexer's manifest-accepted and the
|
||||
// condition keeper's condition-cleared.
|
||||
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
|
||||
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
|
||||
seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs}
|
||||
hear(eventsStream, delivery.ArtifactsGoneEvent+", what wakes a floor, the registry seats' words and manifest-reads-changed are not heard until the builder starts again", func(m jetstream.Msg) {
|
||||
if reads.heard(m, logger.Printf) || floor.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
|
||||
return
|
||||
}
|
||||
heardGone(ctx, b, m, logger.Printf)
|
||||
})
|
||||
if stopped != nil {
|
||||
return stopped
|
||||
}
|
||||
reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "builder_first_views reads nothing and builder_first_view_again is not heard", logger.Printf))
|
||||
// The modules, once their first views and the manifests' are answered: what a module builds is read from the manifests.
|
||||
go func() { errs <- buildFromModules(ctx, b, modules, manifests, candidates) }()
|
||||
// A package's name heard in the indexer's packages looks at its module again, so a done build is numbered then.
|
||||
go func() { errs <- followPackages(ctx, packages, b, logger.Printf) }()
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, acts, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
})
|
||||
// A tag ask or a branch's removal by hand (builder_tag, builder_untag) is written to tag-asks and answered there
|
||||
// (design 51 *Tagged versions*); the checker's tag-wanted and the branch's end join it on their consumers (DLV-B6).
|
||||
tagAsksKV.Ready(func(kv jetstream.KeyValue) {
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
tagAsked(ctx, acts, kv, e, logger.Printf)
|
||||
if !opOf(e) {
|
||||
bounds.disarm(ctx, e.Key())
|
||||
} else if t, err := versions.DecodeTagAsk(e.Value()); err == nil {
|
||||
bounds.asked(ctx, e.Key(), t)
|
||||
}
|
||||
}
|
||||
if tw, err := settle(ctx, kv, each); err != nil {
|
||||
process.NotWatched(logger.Printf, versions.TagAsksBucket, err, "builder_tag and builder_untag are not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, tw, each) }()
|
||||
}
|
||||
})
|
||||
pubBoundsKV.Ready(func(kv jetstream.KeyValue) {
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if versions.IsOldFormBound(e.Key()) {
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
reads.stepWindow() // a node's seven days ended: the step raises old-form-held now
|
||||
}
|
||||
return
|
||||
}
|
||||
pubs.ended(ctx, e)
|
||||
}
|
||||
watchBounds(ctx, kv, each, found.notHeard(ctx), errs, logger.Printf)
|
||||
})
|
||||
boundsKV.Ready(func(kv jetstream.KeyValue) {
|
||||
each := func(e jetstream.KeyValueEntry) { bounds.ended(ctx, e) }
|
||||
if bw, err := settle(ctx, kv, each); err != nil {
|
||||
process.NotWatched(logger.Printf, versions.TagAskBoundsBucket, err, "an unanswered tag ask is not narrated until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, bw, each) }()
|
||||
}
|
||||
})
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
})
|
||||
// A branch's end removes its tagged versions (design 51 *Tagged versions*, DLV-B6): the forge bridge's merged, a pull's
|
||||
// closed or branch-deleted on the builder's durable consumer of its stream; each forge message also wakes a floor whose
|
||||
// bridge_tags was silent. The checker's tag-wanted is kept in tag-asks. A consumer not there yet is said.
|
||||
ends := &branchEnds{acts: acts, findings: found, tagAsks: tagAsksKV.KV, logf: logger.Printf, now: time.Now,
|
||||
wake: func(ctx context.Context) {
|
||||
(&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "")
|
||||
}}
|
||||
for _, c := range []struct {
|
||||
stream, what string
|
||||
hear func(jetstream.Msg)
|
||||
}{
|
||||
{forgeStream, "a branch's end", func(m jetstream.Msg) { ends.heard(ctx, m) }},
|
||||
{checkerAsksStream, "the checker's tag-wanted", func(m jetstream.Msg) {
|
||||
tagWanted(ctx, tagAsksKV.KV(), m, delivery.CheckerModule, time.Now(), logger.Printf)
|
||||
}},
|
||||
} {
|
||||
hear(c.stream, c.what+" is not heard until the builder starts again", c.hear)
|
||||
}
|
||||
if stopped != nil {
|
||||
return stopped
|
||||
}
|
||||
// The state engine-reads, new with ADR 0618 (issue 1790's kind): until it is on the bus each statement is kept in memory
|
||||
// only, and builder_engine_reads answers nothing.
|
||||
process.AwaitBucket(ctx, js, conn, engineReadsBucket, "what each node's engine reads is kept in memory only", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
reads.bind(ctx, kv, errs, logger.Printf)
|
||||
})
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-placementErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// The builder's own state of its first views (views.StatusState); its findings (findingsBucket) are where the condition
|
||||
// keeper raises a first view unanswered past its bound (ADR 0615 §6).
|
||||
var firstViewsBucket = delivery.BuilderModule + "_" + views.StatusState
|
||||
|
||||
// The indexer's states the builder follows (delivery.ModulesBucket, ManifestsBucket, CandidatesBucket), and the owners'
|
||||
// views families as their manifests' `streams` name them.
|
||||
const (
|
||||
indexerViews = "indexer-views"
|
||||
assignerViews = "assigner-views"
|
||||
modulesState = "modules"
|
||||
manifestsState = "manifests"
|
||||
candidatesState = "candidates"
|
||||
)
|
||||
|
||||
// buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and
|
||||
// the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them.
|
||||
func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *views.Picture) error {
|
||||
for _, p := range []*views.Picture{modules, manifests, candidates} {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-p.Follower().Ready():
|
||||
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
|
||||
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
|
||||
default:
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
@@ -488,12 +270,47 @@ func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests,
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
if candidates != nil {
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
}
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
}
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-assignErrs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
}
|
||||
go func() { _ = follow(ctx, cw, eachCandidate) }()
|
||||
return follow(ctx, w, each)
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/forms"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// oldForm keeps the old reference form's window (design 51 *Every build keeps the old reference form*, DLV-B9) in the
|
||||
// state old-form: stepped on every change of what the counted nodes state, of the placement, on a hand drop and at each
|
||||
// build key written, never on a timer. A node holding it seven days is the finding old-form-held; its close is said once
|
||||
// as version.old-form-closed and in the builder's log.
|
||||
type oldForm struct {
|
||||
forms forms.ArtifactForms
|
||||
findings *findings
|
||||
// said emits version.old-form-closed.
|
||||
said func(versions.OldFormClosedBody)
|
||||
// log writes the close into the builder's build log (build.said); nil writes none.
|
||||
log func(delivery.BuildLogEntry)
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
// bounds is the builder's state bounds: each node holding the window has a key there whose time to live ends at its
|
||||
// seven days, so old-form-held is raised then, not at the next change (review round 1); nil keeps none.
|
||||
bounds func() jetstream.KeyValue
|
||||
|
||||
mu sync.Mutex
|
||||
w versions.OldFormWindow
|
||||
kv jetstream.KeyValue
|
||||
// overdue are the nodes the finding is raised for now.
|
||||
overdue []string
|
||||
}
|
||||
|
||||
// bind takes the kept window in, so a restarted builder keeps each node's since and the nodes dropped by hand, and keeps
|
||||
// every later step in kv.
|
||||
func (o *oldForm) bind(ctx context.Context, kv jetstream.KeyValue) {
|
||||
e, err := kv.Get(ctx, versions.OldFormKey)
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
o.logf("%s.%s cannot be read, so the window starts again: %v", versions.OldFormState, versions.OldFormKey, err)
|
||||
default:
|
||||
w, err := versions.DecodeOldFormWindow(e.Value())
|
||||
if err != nil {
|
||||
o.logf("%v; the window starts again", err)
|
||||
break
|
||||
}
|
||||
// Taken in under what was stepped before the state was bound: the kept since and drops are the older truth.
|
||||
for n, s := range o.w.Since {
|
||||
if _, ok := w.Since[n]; !ok && w.Open && slices.Contains(w.Holding, n) {
|
||||
w.Since[n] = s
|
||||
}
|
||||
}
|
||||
for _, d := range o.w.Dropped {
|
||||
w, _ = w.Drop(d.Node, d.By, d.Why, d.At)
|
||||
}
|
||||
o.w = w
|
||||
}
|
||||
o.kv = kv
|
||||
o.keep(ctx)
|
||||
}
|
||||
|
||||
// dropped is the nodes dropped by hand.
|
||||
func (o *oldForm) dropped() map[string]bool {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.w.DroppedSet()
|
||||
}
|
||||
|
||||
// step moves the window to what counted states.
|
||||
// named is every node the read named (the placement's nodes before any is dropped), counted the statements of those
|
||||
// still counted: a read naming no node judges nothing (delivery.StepWindow), so it never closes the window.
|
||||
func (o *oldForm) step(ctx context.Context, named []string, counted map[string][]string) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
before := o.w
|
||||
next, closed, overdue := o.w.Step(named, versions.Holding(o.forms, counted, o.w.DroppedSet()), o.now())
|
||||
o.w = next
|
||||
if !slices.Equal(before.Holding, next.Holding) || before.Open != next.Open || closed {
|
||||
o.keep(ctx)
|
||||
}
|
||||
for _, n := range o.overdue {
|
||||
if !slices.Contains(overdue, n) {
|
||||
o.findings.clear(ctx, versions.OldFormHeld, health.ScopeMachine, n)
|
||||
}
|
||||
}
|
||||
for _, n := range overdue {
|
||||
o.findings.raise(ctx, finding{kind: versions.OldFormHeld, scope: health.ScopeMachine, id: n,
|
||||
summary: fmt.Sprintf("the node-engine of %s has read only an older reference form since %s, so every build key still carries it",
|
||||
n, next.Since[n].Format(time.RFC3339)),
|
||||
headline: "A node holds the old reference form",
|
||||
explanation: "The builder writes an older reference form into every build key while a counted node's engine reads none of the newest (ADR 0624 §1); this node has held it seven days.",
|
||||
resolved: "The node's engine states the newest form, the node leaves the placement, or it is dropped as retired.",
|
||||
needs: "builder_old_form shows the window; a retired node is dropped there by hand; builder_engine_reads asks the node again."})
|
||||
}
|
||||
o.overdue = overdue
|
||||
o.keepBounds(ctx, before, next, overdue)
|
||||
if closed {
|
||||
body := next.Closing()
|
||||
line := fmt.Sprintf("the old reference form's window closed at %s: no counted node-engine reads only an older form (last held by %s; dropped by hand: %d); new build keys leave it out",
|
||||
body.At.Format(time.RFC3339), strings.Join(body.LastHolding, ", "), len(body.Dropped))
|
||||
o.logf("%s", line)
|
||||
// In the build log too (design 51 *The node-engine's own build*), under the builder's own key, the window's key
|
||||
// standing for the fingerprint: the close is no one build's.
|
||||
if o.log != nil {
|
||||
o.log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: delivery.BuilderModule, FP: versions.OldFormKey,
|
||||
At: body.At.UTC(), Line: line, Why: versions.SaidOldFormClosed})
|
||||
}
|
||||
if o.said != nil {
|
||||
o.said(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keepBounds writes a bound for each node holding the window and not yet overdue, ending at its seven days, and deletes
|
||||
// the bound of each node that no longer holds it. A standing bound keeps its time (a Create). Called with mu held.
|
||||
func (o *oldForm) keepBounds(ctx context.Context, before, next versions.OldFormWindow, overdue []string) {
|
||||
if o.bounds == nil {
|
||||
return
|
||||
}
|
||||
kv := o.bounds()
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
for _, n := range before.Holding {
|
||||
if !slices.Contains(next.Holding, n) {
|
||||
if err := kv.Delete(ctx, versions.OldFormBoundKey(n)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
o.logf("the old form's bound of %s is not deleted: %v", n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range next.Holding {
|
||||
left := next.Since[n].Add(versions.FindingAfter).Sub(o.now())
|
||||
if slices.Contains(overdue, n) || left <= 0 {
|
||||
continue
|
||||
}
|
||||
if left < time.Second {
|
||||
left = time.Second // the bus's least time to live
|
||||
}
|
||||
if _, err := kv.Create(ctx, versions.OldFormBoundKey(n), versions.EncodeBound("the seven days "+n+" holds the old form's window", o.now().Add(left)), jetstream.KeyTTL(left)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
o.logf("the old form's bound of %s is not written, so its seven days are seen at the next change only: %v", n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drop is builder_old_form's drop: the node leaves the count, the window is stepped by the caller.
|
||||
func (o *oldForm) drop(ctx context.Context, node, by, why string, at time.Time) (string, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
w, err := o.w.Drop(node, by, why, at)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
o.w = w
|
||||
o.keep(ctx)
|
||||
return node + " dropped from the old form's count by hand; the window is stepped now", nil
|
||||
}
|
||||
|
||||
// keep writes the window, once the state is bound; until then it is kept here only. Called with mu held.
|
||||
func (o *oldForm) keep(ctx context.Context) {
|
||||
if o.kv == nil {
|
||||
return
|
||||
}
|
||||
raw, err := json.Marshal(o.w)
|
||||
if err == nil {
|
||||
_, err = o.kv.Put(ctx, versions.OldFormKey, raw)
|
||||
}
|
||||
if err != nil {
|
||||
o.logf("%s.%s cannot be written: %v", versions.OldFormState, versions.OldFormKey, err)
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/heard"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// publishes raises and clears the builder's findings of a publish that ended badly (versions/publish.go): a holder's
|
||||
// refused, a version unanswered past its bound, kept as a key of the state bounds with its own time to live, and a seat's
|
||||
// outcome that does not read.
|
||||
type publishes struct {
|
||||
bounds func() jetstream.KeyValue
|
||||
versions versions.Versions
|
||||
findings *findings
|
||||
bound time.Duration
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// arm writes the version's bound when none stands, so a standing bound keeps its time. A bound that cannot be written
|
||||
// would never end, so the version is publish-unanswered at once (review round 2): the finding says why.
|
||||
func (p *publishes) arm(ctx context.Context, e versions.Entry) {
|
||||
kv := p.bounds()
|
||||
var err error
|
||||
if kv == nil {
|
||||
err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
|
||||
} else if _, err = kv.Create(ctx, versions.PublishBoundKey(e.Name, e.Version), versions.EncodeBound("the publish of "+e.Name+"@"+e.Version+" to "+e.Seat, time.Now().Add(p.bound)), jetstream.KeyTTL(p.bound)); errors.Is(err, jetstream.ErrKeyExists) {
|
||||
err = nil
|
||||
}
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
p.logf("the publish bound of %s@%s is not written, so it is raised unanswered now: %v", e.Name, e.Version, err)
|
||||
p.unanswered(ctx, e, fmt.Sprintf("its bound could not be written (%v), so the builder cannot wait on the holder's word", err))
|
||||
}
|
||||
|
||||
// unanswered raises publish-unanswered about e's name, saying why.
|
||||
func (p *publishes) unanswered(ctx context.Context, v versions.Entry, why string) {
|
||||
p.findings.raise(ctx, finding{kind: versions.PublishUnanswered, scope: health.ScopeModule, id: v.Name,
|
||||
summary: fmt.Sprintf("%s said neither published nor refused of %s@%s: %s", v.Seat, v.Name, v.Version, why),
|
||||
headline: "A registry seat did not answer a publish",
|
||||
explanation: "The builder built and numbered " + v.Name + " " + v.Version + " and asked " + v.Seat + " to publish it; no word of its holder is known, so the build says done while the version may not be in the registry.",
|
||||
resolved: "That version is published, or a later version of " + v.Name + " is.",
|
||||
needs: "a look at the holder of " + v.Seat + " (its health and log); builder_publish_again asks the publish again."})
|
||||
}
|
||||
|
||||
// heard is told each holder's word the numberer took in: published clears the name's findings, refused raises
|
||||
// publish-refused with the holder's reason, and either ends the version's bound.
|
||||
func (p *publishes) heard(ctx context.Context, seat, event string, o versions.Outcome) {
|
||||
if o.Version != "" {
|
||||
if kv := p.bounds(); kv != nil {
|
||||
if err := kv.Delete(ctx, versions.PublishBoundKey(o.Name, o.Version)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.logf("the publish bound of %s@%s is not ended: %v", o.Name, o.Version, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch event {
|
||||
case versions.HeardPublished:
|
||||
p.findings.clear(ctx, versions.PublishRefused, health.ScopeModule, o.Name)
|
||||
p.findings.clear(ctx, versions.PublishUnanswered, health.ScopeModule, o.Name)
|
||||
case versions.HeardRefused:
|
||||
reason := o.Reason
|
||||
if reason == "" {
|
||||
reason = "it gave no reason"
|
||||
}
|
||||
p.findings.raise(ctx, finding{kind: versions.PublishRefused, scope: health.ScopeModule, id: o.Name,
|
||||
summary: fmt.Sprintf("%s refused to publish %s@%s (digest %s): %s", seat, o.Name, o.Version, o.Digest, reason),
|
||||
headline: "A registry seat refused a publish",
|
||||
explanation: "The builder built and numbered " + o.Name + " " + o.Version + ", and the holder of " + seat + " refused to store it, so that version is not in the registry.",
|
||||
resolved: "That version is published, or a later version of " + o.Name + " is.",
|
||||
needs: "builder_versions shows the version and why; builder_publish_again publishes it again once the cause is mended."})
|
||||
}
|
||||
}
|
||||
|
||||
// unread raises seat-outcome-unread about seat: what it said on subject is no outcome. read clears it.
|
||||
func (p *publishes) unread(ctx context.Context, seat, subject string, why error) {
|
||||
p.findings.raise(ctx, finding{kind: versions.SeatOutcomeUnread, scope: health.ScopeModule, id: seat,
|
||||
summary: fmt.Sprintf("%s said on %s what is no outcome of a publish: %v", seat, subject, why),
|
||||
headline: "A registry seat said what the builder cannot read",
|
||||
explanation: "The builder hears each registry seat's published, refused and removed to move a numbered version; this one does not read, so the version it was about does not move.",
|
||||
resolved: "The next outcome of " + seat + " reads.",
|
||||
needs: "a look at the seat's holder: the outcome's body is its contract's (mesh-contracts)."})
|
||||
}
|
||||
|
||||
func (p *publishes) read(ctx context.Context, seat string) {
|
||||
p.findings.clear(ctx, versions.SeatOutcomeUnread, health.ScopeModule, seat)
|
||||
}
|
||||
|
||||
// ended is told each change of the state bounds: a bound whose time ended (the bus's marker) on a version still numbered
|
||||
// raises publish-unanswered.
|
||||
func (p *publishes) ended(ctx context.Context, e jetstream.KeyValueEntry) {
|
||||
key, ok := versions.PublishBoundOf(e.Key())
|
||||
if e.Operation() != jetstream.KeyValuePurge || !ok {
|
||||
return
|
||||
}
|
||||
raw, _, found, err := p.versions.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
v, err := versions.Decode(raw)
|
||||
if err != nil || v.State != versions.Numbered {
|
||||
return
|
||||
}
|
||||
p.unanswered(ctx, v, fmt.Sprintf("nothing within %s of its build", p.bound))
|
||||
}
|
||||
|
||||
// watchBounds watches the state bounds, handing each change to each: a watch that cannot start loses every bound's end,
|
||||
// so it is the finding family-not-heard naming the state until a later start watches it (review round 2).
|
||||
func watchBounds(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyValueEntry), w heard.Writer, errs chan<- error, logf func(string, ...any)) {
|
||||
const consequence = "a publish no holder answers, and a node holding the old form seven days, are not raised until the builder starts again"
|
||||
pw, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
process.NotWatched(logf, versions.BoundsBucket, err, consequence)
|
||||
w.NotHeard(versions.BoundsState, consequence, err)
|
||||
return
|
||||
}
|
||||
w.Heard(versions.BoundsState)
|
||||
go func() { errs <- follow(ctx, pw, each) }()
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
)
|
||||
|
||||
// EngineReadsState is the builder's state of what each counted node's engine states it reads (novox/hq ADR 0618 §2, ADR
|
||||
// 0624 §1): one key `<node>`, written from the node-engine seat's verb manifest-reads at the builder's start, on the
|
||||
// seat's event manifest-reads-changed for that node, and when a person asks again (builder_engine_reads); a node that
|
||||
// is away keeps its last statement, and its key is removed when the placement no longer names the node. A counted node
|
||||
// with no key reads as stating the baseline: its first forms. No node's record is read in its place.
|
||||
const EngineReadsState = "engine-reads"
|
||||
|
||||
// engineReadsBucket is that state's bucket.
|
||||
const engineReadsBucket = delivery.BuilderModule + "_" + EngineReadsState
|
||||
|
||||
// engineReads keeps the statements and the counted nodes for the builder's forms (build.Builder.Forms).
|
||||
type engineReads struct {
|
||||
keeper *enginereads.Keeper
|
||||
gate *buildrun.ToolchainGate
|
||||
mu sync.Mutex
|
||||
counted []string
|
||||
// window is the old reference form's window over these statements (DLV-B9); nil keeps none.
|
||||
window *oldForm
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// newEngineReads asks through call (process.Conn.CallOn) and keeps each answer in the state once it is bound.
|
||||
func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, logf func(string, ...any)) *engineReads {
|
||||
r := &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf}}
|
||||
r.keeper.Changed = r.stepWindow
|
||||
return r
|
||||
}
|
||||
|
||||
// stepWindow moves the old form's window to what the counted nodes state now (design 51 *Every build keeps the old
|
||||
// reference form*): on a statement's change, the placement's and a hand drop, and at each build key written.
|
||||
func (r *engineReads) stepWindow() {
|
||||
if r.window == nil {
|
||||
return
|
||||
}
|
||||
ctx := r.ctx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
_, counted := r.forms()
|
||||
r.window.step(ctx, r.named(), counted)
|
||||
}
|
||||
|
||||
// named are the nodes the placement names, none dropped: what the window's step judges a read by.
|
||||
func (r *engineReads) named() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return slices.Clone(r.counted)
|
||||
}
|
||||
|
||||
// bind keeps the statements in kv and hears it: every node's last statement at the start, a person's ask again and drop
|
||||
// later. Bound once the state is on the bus (process.AwaitBucket).
|
||||
func (r *engineReads) bind(ctx context.Context, kv jetstream.KeyValue, errs chan<- error, logf func(string, ...any)) {
|
||||
r.keeper.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
r.keeper.Delete = func(node string) error { return kv.Delete(ctx, node) }
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
r.keeper.Heard(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut)
|
||||
}
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
process.NotWatched(logf, engineReadsBucket, err, "builder_engine_reads is not answered until the builder starts again")
|
||||
return
|
||||
}
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
// Every counted node asked again, so what was asked before the state was bound is kept in it too.
|
||||
r.mu.Lock()
|
||||
nodes := slices.Clone(r.counted)
|
||||
r.mu.Unlock()
|
||||
for _, node := range nodes {
|
||||
r.keeper.AskNode(node, "the state engine-reads is bound")
|
||||
}
|
||||
}
|
||||
|
||||
// placed takes the placement in: a node it names newly is asked, one it no longer names is forgotten, key and all. The
|
||||
// first call asks every counted node (the builder's start).
|
||||
func (r *engineReads) placed() {
|
||||
nodes := r.gate.Nodes()
|
||||
r.mu.Lock()
|
||||
before, first := r.counted, r.counted == nil
|
||||
r.counted = nodes
|
||||
r.mu.Unlock()
|
||||
for _, node := range nodes {
|
||||
if first || !slices.Contains(before, node) {
|
||||
r.keeper.AskNode(node, "counted: the placement names it")
|
||||
}
|
||||
}
|
||||
for _, node := range before {
|
||||
if !slices.Contains(nodes, node) {
|
||||
r.keeper.Forget(node)
|
||||
}
|
||||
}
|
||||
r.stepWindow()
|
||||
}
|
||||
|
||||
// heard takes an event of the builder's consumer on EVENTS: manifest-reads-changed asks its node again, and answers true.
|
||||
func (r *engineReads) heard(m jetstream.Msg, logf func(string, ...any)) bool {
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
if !ours {
|
||||
return false
|
||||
}
|
||||
if ackErr := m.Ack(); ackErr != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), ackErr)
|
||||
}
|
||||
if err != nil {
|
||||
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
|
||||
}
|
||||
r.keeper.AskNode(node, enginereads.Event)
|
||||
return true
|
||||
}
|
||||
|
||||
// forms is the counted nodes' statements, and those of the nodes whose engine answered; a retired node dropped by hand
|
||||
// (builder_old_form) counts no more.
|
||||
func (r *engineReads) forms() (present, counted map[string][]string) {
|
||||
r.mu.Lock()
|
||||
nodes := slices.Clone(r.counted)
|
||||
r.mu.Unlock()
|
||||
if r.window != nil {
|
||||
dropped := r.window.dropped()
|
||||
nodes = slices.DeleteFunc(nodes, func(n string) bool { return dropped[n] })
|
||||
}
|
||||
return r.keeper.Present(nodes), r.keeper.Reads(nodes)
|
||||
}
|
||||
|
||||
// built is build.Builder.Forms: forms, and the window stepped with them, so a node's seven days are seen at the next
|
||||
// build key at the latest.
|
||||
func (r *engineReads) built() (present, counted map[string][]string) {
|
||||
present, counted = r.forms()
|
||||
if r.window != nil {
|
||||
ctx := r.ctx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
r.window.step(ctx, r.named(), counted)
|
||||
}
|
||||
return present, counted
|
||||
}
|
||||
@@ -8,31 +8,21 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
|
||||
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
|
||||
}
|
||||
|
||||
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
|
||||
// longer for the builder's answer.
|
||||
const buildAgainBound = 15 * time.Second
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
@@ -44,20 +34,6 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeBuild {
|
||||
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
|
||||
go builtAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeAnnounce {
|
||||
announcedAgain(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
@@ -77,111 +53,23 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
|
||||
if err := emitEvent(ctx, js, cred, build.AskedAgainEvent, "asked-again-"+e.New, e); err != nil {
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
|
||||
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
|
||||
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
case res.Failed != "":
|
||||
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
|
||||
default:
|
||||
r.Pending = res.Revision == 0
|
||||
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
|
||||
if res.Pending != "" {
|
||||
r.Answer = res.Pending
|
||||
if res.Revision != 0 {
|
||||
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
|
||||
// unchanged, and the value is written back with the new revision, or why nothing was done.
|
||||
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
|
||||
if res.Refused != "" {
|
||||
r.Answer, r.Refused = res.Refused, true
|
||||
} else {
|
||||
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key
|
||||
// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential.
|
||||
// The id makes a repeated append of one entry one entry.
|
||||
func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
|
||||
data, err := delivery.EncodeBuildLogEntry(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := delivery.BuildLogKey(e.Module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key)
|
||||
msg.Data = data
|
||||
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// versionActed answers a reasks value of builder_publish_again, builder_floor_again or builder_old_form's drop, a hand act said in the log.
|
||||
func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
|
||||
var err error
|
||||
if r.Mode == build.ModePublishAgain {
|
||||
var ag versions.Again
|
||||
ag, r.Answer, err = acts.PublishAgain(ctx, r.Name, r.FP, r.Why, r.By, r.At)
|
||||
r.Do, r.Version = ag.Do, ag.Entry.Version
|
||||
} else if r.Mode == build.ModeOldFormDrop {
|
||||
if acts.DropOldForm == nil {
|
||||
err = errors.New("this builder keeps no old-form window")
|
||||
} else {
|
||||
r.Answer, err = acts.DropOldForm(ctx, r.Name, r.By, r.Why, r.At)
|
||||
}
|
||||
} else {
|
||||
r.Answer, err = acts.FloorAgain(ctx, r.Name)
|
||||
}
|
||||
if err != nil {
|
||||
r.Answer, r.Refused = err.Error(), true
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: %s of %s by %s (%s): %s", e.Key(), r.Mode, r.Name, r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,528 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/heard"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/enginereads"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/forms"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
|
||||
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
|
||||
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
|
||||
// issue 774).
|
||||
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
|
||||
t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
|
||||
t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard)
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
|
||||
}
|
||||
|
||||
// theCountedNodesEngineReads: the builder asks the node-engine seat's verb of every node the placement names; a node away
|
||||
// and never asked has no statement and reads as the baseline's first forms, with no node record read in its place; an
|
||||
// away node keeps its last statement; a node the placement no longer names leaves the count, its key removed; and
|
||||
// builder_engine_reads drops one node's key.
|
||||
func theCountedNodesEngineReads(t *testing.T) {
|
||||
gate := buildrun.NewToolchainGate(nil, func(string, string, []byte, bool) bool { return false }, t.Logf)
|
||||
place := func(nodes ...string) {
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
pl := delivery.Placement{Node: node}
|
||||
if slices.Contains(nodes, node) {
|
||||
pl.Modules = []delivery.PlacedModule{{Module: "web"}}
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
gate.Placement(key, raw, false, time.Now())
|
||||
}
|
||||
if !slices.Contains(nodes, node) {
|
||||
key, _ := delivery.PlacementModuleKey(node, "web")
|
||||
gate.Placement(key, nil, true, time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
away := map[string]bool{"n2": true}
|
||||
asked := map[string]int{}
|
||||
r := newEngineReads(func(node, key string, _ any) (json.RawMessage, error) {
|
||||
asked[node]++
|
||||
if away[node] {
|
||||
return nil, nats.ErrNoResponders
|
||||
}
|
||||
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
|
||||
}, gate, t.Logf)
|
||||
r.keeper.Sync = true
|
||||
kept := map[string][]byte{}
|
||||
r.keeper.Put = func(node string, v []byte) error { kept[node] = v; return nil }
|
||||
r.keeper.Delete = func(node string) error { delete(kept, node); return nil }
|
||||
place("n1", "n2")
|
||||
r.placed()
|
||||
present, counted := r.forms()
|
||||
if !slices.Equal(present["n1"], []string{"form:new"}) || len(present) != 1 || len(counted) != 2 || counted["n2"] != nil {
|
||||
t.Errorf("away and never asked: present %v, counted %v", present, counted)
|
||||
}
|
||||
if s := r.keeper.Statements()["n2"]; s.From != enginereads.FromNone || s.Answered || asked["n2"] != 1 {
|
||||
t.Errorf("n2, away and never asked, is %+v (asked %d)", s, asked["n2"])
|
||||
}
|
||||
// n2 back, stating {word:module}, then away again: it keeps that statement.
|
||||
away["n2"] = false
|
||||
r.keeper.Heard("n2", mustStatement(t, enginereads.Statement{Node: "n2", From: enginereads.FromVerb, Answered: true,
|
||||
AskAgain: &enginereads.Again{By: "n1.ada", Why: "back"}}), false)
|
||||
away["n2"] = true
|
||||
r.keeper.AskNode("n2", "its event")
|
||||
if _, counted = r.forms(); !slices.Equal(counted["n2"], []string{"form:new"}) || r.keeper.Statements()["n2"].Answered {
|
||||
t.Errorf("away again, n2 reads %v (%+v)", counted["n2"], r.keeper.Statements()["n2"])
|
||||
}
|
||||
// n2 retired: it leaves the placement, and its key goes.
|
||||
place("n1")
|
||||
r.placed()
|
||||
if _, counted = r.forms(); len(counted) != 1 || kept["n2"] != nil {
|
||||
t.Errorf("retired n2: counted %v, its key %s", counted, kept["n2"])
|
||||
}
|
||||
// builder_engine_reads drops one key.
|
||||
st := &memReads{v: map[string][]byte{"n1": kept["n1"]}}
|
||||
tool := enginereads.Tool("builder_engine_reads", "the builder", st, true, time.Now)
|
||||
if _, err := tool.Run(map[string]any{"node": "n1", "drop": "true", "why": "a test"}); err != nil || st.v["n1"] != nil {
|
||||
t.Errorf("drop: %v, the key %s", err, st.v["n1"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustStatement(t *testing.T, s enginereads.Statement) []byte {
|
||||
t.Helper()
|
||||
raw, err := enginereads.Encode(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// memReads is the builder's state of engine reads in memory, as the tool reads it.
|
||||
type memReads struct{ v map[string][]byte }
|
||||
|
||||
func (m *memReads) Keys() ([]string, error) { return slices.Collect(maps.Keys(m.v)), nil }
|
||||
func (m *memReads) Get(key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := m.v[key]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (m *memReads) Put(key string, value []byte) (uint64, error) { m.v[key] = value; return 2, nil }
|
||||
func (m *memReads) Delete(key string) error { delete(m.v, key); return nil }
|
||||
func (m *memReads) Await(string, uint64, func([]byte) bool, time.Duration) ([]byte, error) {
|
||||
return nil, errors.New("no part answers here")
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// wakeMsg is a message of the builder's events consumer: its subject and body, acknowledged.
|
||||
type wakeMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
data []byte
|
||||
acked bool
|
||||
naked bool
|
||||
delayed time.Duration
|
||||
}
|
||||
|
||||
func (m *wakeMsg) Subject() string { return m.subject }
|
||||
func (m *wakeMsg) Data() []byte { return m.data }
|
||||
func (m *wakeMsg) Ack() error { m.acked = true; return nil }
|
||||
func (m *wakeMsg) Nak() error { m.naked = true; return nil }
|
||||
func (m *wakeMsg) NakWithDelay(d time.Duration) error {
|
||||
m.delayed = d
|
||||
return nil
|
||||
}
|
||||
func (m *wakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return nil, errors.New("no metadata")
|
||||
}
|
||||
func (m *wakeMsg) Headers() nats.Header {
|
||||
return nil
|
||||
}
|
||||
|
||||
func aFloorsWakeIsHeard(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
waits := &versions.FloorWaits{}
|
||||
waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0))
|
||||
waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
|
||||
f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf}
|
||||
accepted := &wakeMsg{subject: manifestAcceptedSubject}
|
||||
if !f.heard(context.Background(), accepted) || !accepted.acked || !strings.Contains(logs.String(), "the floor of m/lib is asked again: manifest-accepted") {
|
||||
t.Fatalf("manifest-accepted: acked %v, log %q", accepted.acked, logs.String())
|
||||
}
|
||||
logs.Reset()
|
||||
cleared := &wakeMsg{subject: conditionClearedSubject, data: []byte(`{"subject":{"scope":"probe","id":"ace/mesh-delivery-forge-bridge"}}`)}
|
||||
if !f.heard(context.Background(), cleared) || !strings.Contains(logs.String(), "the floor of m/img is asked again: condition-cleared") {
|
||||
t.Fatalf("condition-cleared: log %q", logs.String())
|
||||
}
|
||||
if f.heard(context.Background(), &wakeMsg{subject: delivery.ArtifactsGoneSubject}) {
|
||||
t.Fatal("artifacts-gone was taken as a floor's wake")
|
||||
}
|
||||
// DLV-B4: a registry seat's outcome on the same consumer moves a version; a published wakes a name whose seat was
|
||||
// silent; one the state cannot take is left for the bus to give again.
|
||||
logs.Reset()
|
||||
waits.Wait("m/pkg", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceSeat}}, nil, time.Unix(0, 0))
|
||||
h := &seatHearer{n: &versions.Numberer{Versions: versions.Versions{Store: noVersions{}}, Waits: waits}, b: &build.Builder{}, logf: log.New(&logs, "", 0).Printf}
|
||||
byDigest := &wakeMsg{subject: "mesh.seat.go-package-registry.event.published", data: []byte(`{"name":"m/other","digest":"sha256:aa","kind":"package","holder":"gitea"}`)}
|
||||
if !h.heard(context.Background(), byDigest) || !byDigest.acked || !strings.Contains(logs.String(), "the floor of m/pkg is asked again: go-package-registry said published") {
|
||||
t.Fatalf("published: acked %v, log %q", byDigest.acked, logs.String())
|
||||
}
|
||||
refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
|
||||
if !h.heard(context.Background(), refused) || refused.acked || !refused.naked {
|
||||
t.Fatalf("a refusal the state cannot take: acked %v, naked %v, log %q", refused.acked, refused.naked, logs.String())
|
||||
}
|
||||
if h.heard(context.Background(), &wakeMsg{subject: manifestAcceptedSubject}) {
|
||||
t.Fatal("manifest-accepted was taken as a seat's outcome")
|
||||
}
|
||||
}
|
||||
|
||||
// noVersions is a state versions that holds nothing.
|
||||
type noVersions struct{}
|
||||
|
||||
func (noVersions) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
func (noVersions) Create(context.Context, string, []byte) (uint64, error) {
|
||||
return 0, errors.New("read only")
|
||||
}
|
||||
func (noVersions) Update(context.Context, string, []byte, uint64) (uint64, error) {
|
||||
return 0, errors.New("read only")
|
||||
}
|
||||
func (noVersions) KeysOf(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func aBranchsEndIsHeard(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
f forgeEnd
|
||||
want bool
|
||||
}{
|
||||
{forgeEnd{Kind: delivery.ForgeMerged, Repository: "o/r", Branch: "feat/x"}, true},
|
||||
{forgeEnd{Kind: forgeBranchDeleted, Repository: "o/r", Branch: "feat/x"}, true},
|
||||
{forgeEnd{Kind: delivery.ForgePull, Repository: "o/r", Branch: "feat/x", Action: delivery.PullClosed}, true},
|
||||
{forgeEnd{Kind: delivery.ForgePull, Repository: "o/r", Branch: "feat/x", Action: delivery.PullOpened}, false},
|
||||
{forgeEnd{Kind: delivery.ForgeMerged, Repository: "o/r"}, false},
|
||||
} {
|
||||
if got := c.f.ended(); got != c.want {
|
||||
t.Errorf("%+v ended = %v, want %v", c.f, got, c.want)
|
||||
}
|
||||
}
|
||||
var logs bytes.Buffer
|
||||
woken := 0
|
||||
h := &branchEnds{acts: versions.Acts{Numberer: &versions.Numberer{Versions: versions.Versions{Store: noVersions{}}}}, findings: &findings{logf: t.Logf},
|
||||
tagAsks: func() jetstream.KeyValue { return nil }, wake: func(context.Context) { woken++ }, logf: log.New(&logs, "", 0).Printf, now: time.Now}
|
||||
open := &wakeMsg{subject: "mesh.mod.mesh-delivery-forge-bridge.event.forge.o.r.pull", data: []byte(`{"kind":"pull","repository":"o/r","branch":"feat/x","action":"opened"}`)}
|
||||
h.heard(context.Background(), open)
|
||||
merged := &wakeMsg{subject: "mesh.mod.mesh-delivery-forge-bridge.event.forge.o.r.merged", data: []byte(`{"kind":"merged","repository":"o/r","branch":"feat/x"}`)}
|
||||
h.heard(context.Background(), merged)
|
||||
if !open.acked || !merged.acked || woken != 2 {
|
||||
t.Errorf("an open pull acked %v, a merge with nothing tagged acked %v, floors woken %d times", open.acked, merged.acked, woken)
|
||||
}
|
||||
// The checker's family holds its verdicts beside tag-wanted: a verdict is acknowledged and not kept; a tag-wanted with
|
||||
// tag-asks not bound is heard again.
|
||||
verdict := &wakeMsg{subject: "mesh.stream.mesh-delivery-checker.event.verdict", data: []byte(`{"repository":"o/r"}`)}
|
||||
wanted := &wakeMsg{subject: tagWantedSubject, data: []byte(`{"repository":"o/r","branch":"feat/x","head":"abc","tag":"feat-x"}`)}
|
||||
for _, m := range []*wakeMsg{verdict, wanted} {
|
||||
tagWanted(context.Background(), nil, m, delivery.CheckerModule, time.Now(), t.Logf)
|
||||
}
|
||||
if !verdict.acked || verdict.delayed != 0 || wanted.acked || wanted.delayed != versions.RemoveAgainAfter || !strings.HasPrefix(tagWantedSubject, "mesh.stream.") {
|
||||
t.Errorf("a verdict acked %v (delayed %v), a tag-wanted acked %v (delayed %v)", verdict.acked, verdict.delayed, wanted.acked, wanted.delayed)
|
||||
}
|
||||
unreadable := &wakeMsg{subject: merged.subject, data: merged.data}
|
||||
h.acts.Numberer.Versions.Store = failingVersions{}
|
||||
h.heard(context.Background(), unreadable)
|
||||
if unreadable.acked || unreadable.delayed != versions.RemoveAgainAfter {
|
||||
t.Errorf("a branch's end the versions cannot answer: acked %v, given again after %v", unreadable.acked, unreadable.delayed)
|
||||
}
|
||||
}
|
||||
|
||||
// failingVersions is a state versions that cannot be read.
|
||||
type failingVersions struct{ noVersions }
|
||||
|
||||
func (failingVersions) KeysOf(context.Context, string) ([]string, error) {
|
||||
return nil, errors.New("down")
|
||||
}
|
||||
|
||||
func aFailedTrunkBuildIsAFinding(t *testing.T) {
|
||||
st := &memFindings{vals: map[string][]byte{}}
|
||||
f := &findings{store: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
ended := func(outcome, ref string) delivery.BuildLogEntry {
|
||||
return delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Ref: ref, Outcome: outcome}
|
||||
}
|
||||
key := health.FindingKey(health.ScopeModule, "m", BuildFailed)
|
||||
f.buildEnded(context.Background(), ended(delivery.BuildFailed, "refs/pull/3/head"))
|
||||
if len(st.vals) != 0 {
|
||||
t.Fatalf("a candidate's failed build is a finding: %v", st.vals)
|
||||
}
|
||||
f.buildEnded(context.Background(), ended(delivery.BuildFailed, ""))
|
||||
got, err := health.DecodeFinding(st.vals[key])
|
||||
if err != nil || got.Observer != delivery.BuilderModule || !strings.Contains(got.Summary, "trunk build of m at f1") {
|
||||
t.Fatalf("the finding = %+v, %v", got, err)
|
||||
}
|
||||
f.buildEnded(context.Background(), ended(delivery.BuildDone, ""))
|
||||
if _, ok := st.vals[key]; ok {
|
||||
t.Error("a later done trunk build leaves build-failed")
|
||||
}
|
||||
// The floor finding: raised while a name waits on its floor, cleared once it is known.
|
||||
fw := &versions.FloorWaits{Said: f.floorSaid(context.Background())}
|
||||
fw.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
|
||||
floorKey := health.FindingKey(health.ScopeModule, "m/lib", versions.FloorSilent)
|
||||
if got, err := health.DecodeFinding(st.vals[floorKey]); err != nil || !strings.Contains(got.Summary, "bridge_tags did not answer") {
|
||||
t.Fatalf("the floor finding = %+v, %v", got, err)
|
||||
}
|
||||
fw.Answered("m/lib")
|
||||
if _, ok := st.vals[floorKey]; ok {
|
||||
t.Error("a known floor leaves floor-silent")
|
||||
}
|
||||
// Design 51 *Health and conditions* (review round 1): a holder's refused is publish-refused with its reason, a version
|
||||
// no holder answered within its bound publish-unanswered, a seat's outcome that does not read seat-outcome-unread;
|
||||
// a later published of the name clears the first two, the seat's next outcome that reads the third.
|
||||
vs := versions.Versions{Store: memVersions{}}
|
||||
for _, v := range []string{"0.1.3", "0.1.4"} {
|
||||
raw, err := versions.Encode(versions.Entry{Name: "m/program", Kind: versions.KindBundle, Series: "0.1", Version: v, Seat: "oci-artifact-registry", State: versions.Numbered})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vs.Store.(memVersions)[versions.Key("m/program", v)] = raw
|
||||
}
|
||||
pubs := &publishes{bounds: func() jetstream.KeyValue { return nil }, versions: vs, findings: f, bound: versions.PublishBound, logf: t.Logf}
|
||||
h := &seatHearer{n: &versions.Numberer{Versions: vs}, b: &build.Builder{}, logf: t.Logf, pub: pubs}
|
||||
refusedKey := health.FindingKey(health.ScopeModule, "m/program", versions.PublishRefused)
|
||||
refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
|
||||
if !h.heard(context.Background(), refused) || !refused.acked {
|
||||
t.Fatalf("a refusal: acked %v", refused.acked)
|
||||
}
|
||||
if got, err := health.DecodeFinding(st.vals[refusedKey]); err != nil || !strings.Contains(got.Summary, "refused to publish m/program@0.1.3") || !strings.Contains(got.Summary, "bytes differ") {
|
||||
t.Fatalf("a holder's refused: the finding = %+v, %v", got, err)
|
||||
}
|
||||
pubs.ended(context.Background(), purged{key: versions.PublishBoundKey("m/program", "0.1.4")})
|
||||
unansweredKey := health.FindingKey(health.ScopeModule, "m/program", versions.PublishUnanswered)
|
||||
if got, err := health.DecodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "neither published nor refused of m/program@0.1.4") {
|
||||
t.Fatalf("a bound ended unanswered: the finding = %+v, %v", got, err)
|
||||
}
|
||||
// Review round 2: a bound that cannot be written is publish-unanswered at once; a watch of bounds that cannot start
|
||||
// is family-not-heard naming the state.
|
||||
delete(st.vals, unansweredKey)
|
||||
pubs.arm(context.Background(), versions.Entry{Name: "m/program", Version: "0.1.4", Seat: "oci-artifact-registry"})
|
||||
if got, err := health.DecodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "its bound could not be written") {
|
||||
t.Fatalf("a bound not written: the finding = %+v, %v", got, err)
|
||||
}
|
||||
watchBounds(context.Background(), refusedWatch{err: errors.New("permissions violation")}, func(jetstream.KeyValueEntry) {}, f.notHeard(context.Background()), nil, t.Logf)
|
||||
if got, err := health.DecodeFinding(st.vals[heard.Key(delivery.BuilderModule, versions.BoundsState)]); err != nil || !strings.Contains(got.Summary, "permissions violation") {
|
||||
t.Fatalf("bounds not watched: the finding = %+v, %v", got, err)
|
||||
}
|
||||
delete(st.vals, heard.Key(delivery.BuilderModule, versions.BoundsState))
|
||||
malformed := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"version":"0.1.4"}`)}
|
||||
unreadKey := health.FindingKey(health.ScopeModule, "oci-artifact-registry", versions.SeatOutcomeUnread)
|
||||
if !h.heard(context.Background(), malformed) || !malformed.acked || st.vals[unreadKey] == nil {
|
||||
t.Fatalf("an outcome that does not read: acked %v, findings %v", malformed.acked, st.vals)
|
||||
}
|
||||
published := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"name":"m/program","version":"0.1.4","digest":"sha256:cc"}`)}
|
||||
if !h.heard(context.Background(), published) || st.vals[refusedKey] != nil || st.vals[unansweredKey] != nil || st.vals[unreadKey] != nil {
|
||||
t.Fatalf("a later published leaves the findings: %v", st.vals)
|
||||
}
|
||||
// ADR 0615 §5 (review round 1): a consumer not there yet is family-not-heard naming its stream, cleared when it binds.
|
||||
notHeard := f.notHeard(context.Background())
|
||||
unheardKey := heard.Key(delivery.BuilderModule, "FORGE")
|
||||
_ = notHeard.Bind("FORGE", "a branch's end is not heard", func() error { return errors.New("consumer not found") })
|
||||
if got, err := health.DecodeFinding(st.vals[unheardKey]); err != nil || got.Kind != heard.Kind || !strings.Contains(got.Summary, "does not hear FORGE") {
|
||||
t.Fatalf("a consumer not there: the finding = %+v, %v", got, err)
|
||||
}
|
||||
_ = notHeard.Bind("FORGE", "a branch's end is not heard", func() error { return nil })
|
||||
if _, ok := st.vals[unheardKey]; ok {
|
||||
t.Error("a consumer bound leaves family-not-heard")
|
||||
}
|
||||
// DLV-B9, the old form's window: one counted node reading only the first form holds it; seven days on it is
|
||||
// old-form-held about that node; the node stating the newest form closes it, said once, the finding cleared; a
|
||||
// retired node dropped by hand leaves the count.
|
||||
now := time.Unix(0, 0)
|
||||
fs := forms.ArtifactForms{{Name: "registry", First: true, Kinds: []string{"image"}}, {Name: "oci", Kinds: []string{"image"}}}
|
||||
var closings []versions.OldFormClosedBody
|
||||
var logged []delivery.BuildLogEntry
|
||||
marks := &memBounds{keys: map[string]bool{}}
|
||||
window := &oldForm{forms: fs, findings: f, logf: t.Logf, now: func() time.Time { return now }, bounds: func() jetstream.KeyValue { return marks },
|
||||
said: func(b versions.OldFormClosedBody) { closings = append(closings, b) },
|
||||
log: func(e delivery.BuildLogEntry) { logged = append(logged, e) }}
|
||||
reads := &engineReads{keeper: &enginereads.Keeper{}, window: window, counted: []string{"a", "b"}}
|
||||
newest := []string{forms.FormNeed("oci")}
|
||||
counted := map[string][]string{"a": newest, "b": nil}
|
||||
window.step(context.Background(), []string{"a", "b"}, counted)
|
||||
heldKey := health.FindingKey(health.ScopeMachine, "b", versions.OldFormHeld)
|
||||
if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || st.vals[heldKey] != nil {
|
||||
t.Fatalf("one node lacking the newest form: %+v", window.w)
|
||||
}
|
||||
// Review round 1: the node holding it has a bound in the state bounds ending at its seven days, so old-form-held is
|
||||
// raised then, not at the next change.
|
||||
if !marks.keys[versions.OldFormBoundKey("b")] || marks.keys[versions.OldFormBoundKey("a")] {
|
||||
t.Fatalf("the old form's bounds = %v, want b's alone", marks.keys)
|
||||
}
|
||||
// Security-upgrade F3 (review round 1): an empty node_haves read, naming no node, leaves the window open.
|
||||
window.step(context.Background(), nil, map[string][]string{})
|
||||
if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || len(closings) != 0 {
|
||||
t.Fatalf("an empty read closed the window: %+v, closings %+v", window.w, closings)
|
||||
}
|
||||
now = now.Add(versions.FindingAfter)
|
||||
window.step(context.Background(), []string{"a", "b"}, counted)
|
||||
if got, err := health.DecodeFinding(st.vals[heldKey]); err != nil || !strings.Contains(got.Summary, "the node-engine of b") {
|
||||
t.Fatalf("seven days held: the finding = %+v, %v", got, err)
|
||||
}
|
||||
counted["b"] = newest
|
||||
window.step(context.Background(), []string{"a", "b"}, counted)
|
||||
window.step(context.Background(), []string{"a", "b"}, counted)
|
||||
if marks.keys[versions.OldFormBoundKey("b")] {
|
||||
t.Error("a node no longer holding the window keeps its bound")
|
||||
}
|
||||
if window.w.Open || len(closings) != 1 || !slices.Equal(closings[0].LastHolding, []string{"b"}) || st.vals[heldKey] != nil {
|
||||
t.Errorf("the node back with the newest form: %+v, closings %+v, finding left %v", window.w, closings, st.vals[heldKey] != nil)
|
||||
}
|
||||
if len(logged) != 1 || logged[0].Event != delivery.BuildSaidEvent || logged[0].Why != versions.SaidOldFormClosed ||
|
||||
!strings.Contains(logged[0].Line, "last held by b") {
|
||||
t.Errorf("the build log = %+v, want the close said once (build.said)", logged)
|
||||
} else if _, err := delivery.EncodeBuildLogEntry(logged[0]); err != nil {
|
||||
t.Errorf("the close is no entry the build log takes: %v", err)
|
||||
}
|
||||
if _, err := window.drop(context.Background(), "b", "op@shanks", "retired", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, c := reads.forms(); len(c) != 1 || c["b"] != nil {
|
||||
t.Errorf("a node dropped by hand is still counted: %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// memVersions is a state versions in memory.
|
||||
type memVersions map[string][]byte
|
||||
|
||||
func (m memVersions) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := m[key]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (m memVersions) Create(_ context.Context, key string, v []byte) (uint64, error) {
|
||||
m[key] = v
|
||||
return 1, nil
|
||||
}
|
||||
func (m memVersions) Update(_ context.Context, key string, v []byte, _ uint64) (uint64, error) {
|
||||
m[key] = v
|
||||
return 2, nil
|
||||
}
|
||||
func (m memVersions) KeysOf(context.Context, string) ([]string, error) {
|
||||
return slices.Collect(maps.Keys(m)), nil
|
||||
}
|
||||
|
||||
// memBounds is the state bounds in memory: which keys stand.
|
||||
type memBounds struct {
|
||||
jetstream.KeyValue
|
||||
keys map[string]bool
|
||||
}
|
||||
|
||||
func (m *memBounds) Create(_ context.Context, key string, _ []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if m.keys[key] {
|
||||
return 0, jetstream.ErrKeyExists
|
||||
}
|
||||
m.keys[key] = true
|
||||
return 1, nil
|
||||
}
|
||||
func (m *memBounds) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
delete(m.keys, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// purged is the bus's marker of a key whose time to live ended.
|
||||
type purged struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
}
|
||||
|
||||
func (p purged) Key() string { return p.key }
|
||||
func (p purged) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePurge }
|
||||
|
||||
// memFindings is a findings bucket in memory.
|
||||
type memFindings struct{ vals map[string][]byte }
|
||||
|
||||
type memEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
v []byte
|
||||
}
|
||||
|
||||
func (e memEntry) Value() []byte { return e.v }
|
||||
|
||||
func (m *memFindings) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := m.vals[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return memEntry{v: v}, nil
|
||||
}
|
||||
func (m *memFindings) Put(_ context.Context, key string, v []byte) (uint64, error) {
|
||||
m.vals[key] = v
|
||||
return 1, nil
|
||||
}
|
||||
func (m *memFindings) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
delete(m.vals, key)
|
||||
return nil
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// seatEvent is one registry seat's event the builder hears: the seat and the event's name.
|
||||
type seatEvent struct{ seat, event string }
|
||||
|
||||
// seatEvents are the four registry seats' published, refused, removed and kept, by subject (design 51 *How a publish ends*,
|
||||
// DLV-B4): each a seat event `<seat>.<event>` on the shared events stream, heard on the builder's durable consumer of
|
||||
// it through the manifest's consumes. The subjects are the seats' contracts' (mesh-contracts).
|
||||
var seatEvents = map[string]seatEvent{
|
||||
containerimageregistry.SubjectPublished: {containerimageregistry.Seat, containerimageregistry.EventPublished},
|
||||
containerimageregistry.SubjectRefused: {containerimageregistry.Seat, containerimageregistry.EventRefused},
|
||||
containerimageregistry.SubjectRemoved: {containerimageregistry.Seat, containerimageregistry.EventRemoved},
|
||||
containerimageregistry.SubjectKept: {containerimageregistry.Seat, containerimageregistry.EventKept},
|
||||
ociartifactregistry.SubjectPublished: {ociartifactregistry.Seat, ociartifactregistry.EventPublished},
|
||||
ociartifactregistry.SubjectRefused: {ociartifactregistry.Seat, ociartifactregistry.EventRefused},
|
||||
ociartifactregistry.SubjectRemoved: {ociartifactregistry.Seat, ociartifactregistry.EventRemoved},
|
||||
ociartifactregistry.SubjectKept: {ociartifactregistry.Seat, ociartifactregistry.EventKept},
|
||||
gopackageregistry.SubjectPublished: {gopackageregistry.Seat, gopackageregistry.EventPublished},
|
||||
gopackageregistry.SubjectRefused: {gopackageregistry.Seat, gopackageregistry.EventRefused},
|
||||
gopackageregistry.SubjectRemoved: {gopackageregistry.Seat, gopackageregistry.EventRemoved},
|
||||
gopackageregistry.SubjectKept: {gopackageregistry.Seat, gopackageregistry.EventKept},
|
||||
npmpackageregistry.SubjectPublished: {npmpackageregistry.Seat, npmpackageregistry.EventPublished},
|
||||
npmpackageregistry.SubjectRefused: {npmpackageregistry.Seat, npmpackageregistry.EventRefused},
|
||||
npmpackageregistry.SubjectRemoved: {npmpackageregistry.Seat, npmpackageregistry.EventRemoved},
|
||||
npmpackageregistry.SubjectKept: {npmpackageregistry.Seat, npmpackageregistry.EventKept},
|
||||
}
|
||||
|
||||
// seatHearer takes the registry seats' outcomes in: each moves a numbered version (versions.Numberer.Heard), and a
|
||||
// `published` that wakes a name waiting on its floor looks at every module again.
|
||||
type seatHearer struct {
|
||||
n *versions.Numberer
|
||||
b *build.Builder
|
||||
logf func(string, ...any)
|
||||
// pub raises and clears the findings of a publish that ended badly; nil raises none.
|
||||
pub *publishes
|
||||
}
|
||||
|
||||
// heard takes a message of the events consumer in when it is a registry seat's outcome, and answers whether it was.
|
||||
// One the state could not take is left unacknowledged (Nak), so the bus gives it again; one that is no outcome is said
|
||||
// and dropped.
|
||||
func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
se, ok := seatEvents[m.Subject()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
var o versions.Outcome
|
||||
if err := json.Unmarshal(m.Data(), &o); err != nil || o.Name == "" {
|
||||
if err == nil {
|
||||
err = errors.New("it names no name")
|
||||
}
|
||||
h.logf("%s holds what is no outcome of %s, left: %v", m.Subject(), se.seat, err)
|
||||
if h.pub != nil {
|
||||
h.pub.unread(ctx, se.seat, m.Subject(), err)
|
||||
}
|
||||
h.ack(m)
|
||||
return true
|
||||
}
|
||||
if h.pub != nil {
|
||||
h.pub.read(ctx, se.seat)
|
||||
}
|
||||
id := ""
|
||||
if m.Headers() != nil {
|
||||
id = m.Headers().Get("x-event-id")
|
||||
}
|
||||
wake, err := h.n.Heard(ctx, se.seat, se.event, id, o)
|
||||
if err != nil {
|
||||
h.logf("%v; heard again", err)
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
h.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
return true
|
||||
}
|
||||
if h.pub != nil {
|
||||
h.pub.heard(ctx, se.seat, se.event, o)
|
||||
}
|
||||
h.ack(m)
|
||||
if len(wake) > 0 {
|
||||
h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event)
|
||||
h.b.Reconsider(ctx)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *seatHearer) ack(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
h.logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// kvKeeper is the state floor-waits as versions.FloorKeeper.
|
||||
type kvKeeper struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (k kvKeeper) Put(key string, value []byte) error {
|
||||
_, err := k.kv.Put(k.ctx, key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (k kvKeeper) Delete(key string) error {
|
||||
err := k.kv.Delete(k.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// heldFloorWaits are the waits floor-waits holds at the builder's start; a value that is no wait is said and left.
|
||||
func heldFloorWaits(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) []versions.FloorWait {
|
||||
l, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
logf("%s cannot be listed, so a restart's waits are asked at their next claim: %v", versions.FloorWaitsState, err)
|
||||
return nil
|
||||
}
|
||||
defer func() { _ = l.Stop() }()
|
||||
var out []versions.FloorWait
|
||||
for k := range l.Keys() {
|
||||
e, err := kv.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
w, err := versions.DecodeFloorWait(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is no floor wait, left: %v", versions.FloorWaitsState, k, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||
// toolchain agreement this file repairs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.raw[module] == nil {
|
||||
g.raw[module] = map[string][]byte{}
|
||||
}
|
||||
if deleted {
|
||||
delete(g.raw[module], key)
|
||||
} else {
|
||||
g.raw[module][key] = value
|
||||
}
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
if !g.known {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||
// passed on changed.
|
||||
func (g *toolchainGate) Settled() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.known = true
|
||||
changed := false
|
||||
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||
changed = g.pass(module, k) || changed
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||
errs := make(chan error, 1)
|
||||
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||
if err != nil {
|
||||
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return errs, nil
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, "typescript")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
|
||||
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||
return tc.Version, none
|
||||
}
|
||||
|
||||
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||
key, value := gateReport(t, node, version)
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||
}
|
||||
// ace reports again while unassigned (an old value seen again): still left out.
|
||||
key, value := gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||
t.Fatal("deleting g14's report passed nothing on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,31 +9,26 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// eventsStream is the stream every module's and role's events are on, where the grants part makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-sdk buscompose EventsStream).
|
||||
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
|
||||
const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held"
|
||||
|
||||
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
|
||||
type toolCaller func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// storedVia is build.Builder's Stored: it asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2, issue 1937)
|
||||
// and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: the key
|
||||
// is then left done, and asked again at the next module value or artifacts-gone.
|
||||
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
|
||||
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
|
||||
// the key is then left done, and the controller's take-in still refuses a gone blob.
|
||||
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
|
||||
return func(_ context.Context, references []string) ([]string, error) {
|
||||
body := map[string]any{"references": references}
|
||||
raw, err := call(SeatHeld, body)
|
||||
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking held: %w", err)
|
||||
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
|
||||
}
|
||||
a, err := delivery.DecodeHeldAnswer(raw, references)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// tagBounds narrates a tag ask the builder has not answered within five minutes, with what it waits on (design 51
|
||||
// *Tagged versions*): each unanswered ask has a key in the state tag-ask-bounds written with its own time to live, and
|
||||
// the bus's marker of its end, heard on the watch, is what narrates it and writes the bound again. No timer.
|
||||
type tagBounds struct {
|
||||
asks func() jetstream.KeyValue
|
||||
bounds func() jetstream.KeyValue
|
||||
waits func() []versions.FloorWait
|
||||
logf func(string, ...any)
|
||||
// log writes a narration into the builder's build log (build.said); nil writes none.
|
||||
log func(delivery.BuildLogEntry)
|
||||
now func() time.Time
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// arm writes key's bound when none stands: a Create, so a standing bound keeps its time and a restart arms each
|
||||
// unanswered ask whose bound ended while the builder was away.
|
||||
func (tb *tagBounds) arm(ctx context.Context, key string) {
|
||||
kv := tb.bounds()
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(tb.bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
tb.logf("the bound of the tag ask %s is not written, so it is not narrated past five minutes: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// disarm removes key's bound: the ask was answered or ended.
|
||||
func (tb *tagBounds) disarm(ctx context.Context, key string) {
|
||||
if kv := tb.bounds(); kv != nil {
|
||||
_ = kv.Delete(ctx, key)
|
||||
}
|
||||
}
|
||||
|
||||
// asked is told each value of tag-asks: an unanswered ask is armed, any other disarmed.
|
||||
func (tb *tagBounds) asked(ctx context.Context, key string, t versions.TagAsk) {
|
||||
if t.Unanswered() {
|
||||
tb.arm(ctx, key)
|
||||
return
|
||||
}
|
||||
tb.disarm(ctx, key)
|
||||
}
|
||||
|
||||
// ended is told each change of tag-ask-bounds: a bound whose time ended (the bus's marker, a purge on the watch) on an
|
||||
// ask still unanswered narrates it and arms it again.
|
||||
func (tb *tagBounds) ended(ctx context.Context, e jetstream.KeyValueEntry) {
|
||||
if e.Operation() != jetstream.KeyValuePurge {
|
||||
return
|
||||
}
|
||||
kv := tb.asks()
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
got, err := kv.Get(ctx, e.Key())
|
||||
if err != nil {
|
||||
return // the ask ended with its branch
|
||||
}
|
||||
t, err := versions.DecodeTagAsk(got.Value())
|
||||
if err != nil || !t.Unanswered() {
|
||||
return
|
||||
}
|
||||
var waits []versions.FloorWait
|
||||
if tb.waits != nil {
|
||||
waits = tb.waits()
|
||||
}
|
||||
line := versions.NarrateUnanswered(t, waits, tb.now())
|
||||
tb.logf("%s", line)
|
||||
// In the build log too (design 51 *Tagged versions*): under the builder's own key, no build being claimed yet, the
|
||||
// ask's key standing for the fingerprint.
|
||||
if tb.log != nil {
|
||||
tb.log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: delivery.BuilderModule, FP: e.Key(), Commit: t.Head,
|
||||
At: tb.now().UTC(), Line: line, Why: versions.SaidUnanswered})
|
||||
}
|
||||
tb.arm(ctx, e.Key())
|
||||
}
|
||||
|
||||
// tagNumbered marks the tag ask of a check's build's head numbered with its versions in tag-asks, by compare-and-set.
|
||||
func tagNumbered(kv func() jetstream.KeyValue, logf func(string, ...any), now func() time.Time) func(context.Context, versions.BuildClaim, versions.Numbering) {
|
||||
return func(ctx context.Context, c versions.BuildClaim, got versions.Numbering) {
|
||||
store := kv()
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
lister, err := store.ListKeysFiltered(ctx, versions.EscapeToken(c.Repository)+".>")
|
||||
if err != nil {
|
||||
logf("the tag ask of %s at %s is not marked numbered: %v", c.Repository, c.Head, err)
|
||||
return
|
||||
}
|
||||
for key := range lister.Keys() {
|
||||
for range 3 {
|
||||
e, err := store.Get(ctx, key)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
t, err := versions.DecodeTagAsk(e.Value())
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
t, changed := t.WithNumbered(c.Module, c.Head, got.Versions, got.Names, now())
|
||||
if !changed {
|
||||
break
|
||||
}
|
||||
data, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if _, err = store.Update(ctx, key, data, e.Revision()); err == nil {
|
||||
logf("%s: tag ask of %s %s: %s", key, t.Repository, t.Branch, t.Answer)
|
||||
break
|
||||
}
|
||||
if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("the tag ask %s is not marked numbered: %v", key, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -1,352 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:<registry>.versions`.
|
||||
const (
|
||||
bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags"
|
||||
requirementsTool = "mesh-delivery-indexer.indexer_requirements"
|
||||
)
|
||||
|
||||
// awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue
|
||||
// 1790); until then every use is an error, so a trunk build is built unnumbered and said (build.Builder.number).
|
||||
type awaitedVersions struct{ b *process.AwaitedBucket }
|
||||
|
||||
var errVersionsMissing = errors.New("the state " + versions.Bucket + " is not on the bus yet")
|
||||
|
||||
func (s awaitedVersions) kv() (jetstream.KeyValue, error) {
|
||||
if kv := s.b.KV(); kv != nil {
|
||||
return kv, nil
|
||||
}
|
||||
return nil, errVersionsMissing
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
// indexerPackages is the indexer's packages state as the builder reads a package's name in its ecosystem from it (design
|
||||
// 32): the picture of its read tool's first view and its views family (ADR 0615 §1), never its bucket. Until the first
|
||||
// view is answered a package is built unnumbered and said (versions.Numberer.Names), and numbered once its name is heard
|
||||
// (followPackages).
|
||||
type indexerPackages struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (s indexerPackages) Keys(ctx context.Context) ([]string, error) {
|
||||
keys, err := s.kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (s indexerPackages) Get(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// followPackages watches the picture of the indexer's packages state: each series key heard that names a package's name
|
||||
// (versions.NamedModule), its first view's included, looks at its module again (build.Builder.PackageNamed), so a done
|
||||
// build that left the package unnumbered is numbered then with its digests. Woken by the views family, never a timer; a
|
||||
// key heard again naming the name and module it named (the indexer writes a series at each publish) looks at nothing.
|
||||
func followPackages(ctx context.Context, kv jetstream.KeyValue, b *build.Builder, logf func(string, ...any)) error {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
heard := map[string]string{}
|
||||
return follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
delete(heard, e.Key())
|
||||
return
|
||||
}
|
||||
module, name, named := versions.NamedModule(e.Key(), e.Value())
|
||||
if named && heard[e.Key()] != module+" "+name {
|
||||
heard[e.Key()] = module + " " + name
|
||||
logf("%s.%s names a package of %s: its build is looked at again", delivery.PackagesBucket, e.Key(), module)
|
||||
b.PackageNamed(ctx, module)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func versionsConflict(err error) error {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return versions.ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Create(ctx, key, value)
|
||||
return rev, versionsConflict(err)
|
||||
}
|
||||
|
||||
func (s awaitedVersions) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Update(ctx, key, value, revision)
|
||||
return rev, versionsConflict(err)
|
||||
}
|
||||
|
||||
func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := kv.ListKeysFiltered(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = l.Stop() }()
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
|
||||
// versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source
|
||||
// silent.
|
||||
func floorVia(call toolCaller) versions.FloorSources {
|
||||
return versions.FloorSources{
|
||||
Seat: func(_ context.Context, seat, name string) ([]string, error) {
|
||||
raw, err := call("seat:"+seat+".versions", map[string]any{"name": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Versions []struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", seat, err)
|
||||
}
|
||||
var out []string
|
||||
for _, v := range a.Versions {
|
||||
if v.Version != "" {
|
||||
out = append(out, v.Version)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("bridge_tags answered what is no list of tags: %w", err)
|
||||
}
|
||||
return a.Tags, nil
|
||||
},
|
||||
Requirements: func(_ context.Context, name string) ([]string, error) {
|
||||
raw, err := call(requirementsTool, map[string]any{"package": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Requirements []string `json:"requirements"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("indexer_requirements answered what is no list of versions: %w", err)
|
||||
}
|
||||
return a.Requirements, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// emitNumbered says the builder's event version.numbered (design 51 *Events of the builder*), on its own family apart
|
||||
// from its builds' ends; the name and version are the bus's dedup id.
|
||||
func emitNumbered(ctx context.Context, js jetstream.JetStream, cred process.Account, b delivery.VersionNumbered) error {
|
||||
id := "numbered-" + versions.Key(b.Name, b.Version)
|
||||
if b.Again != "" {
|
||||
id += "-again-" + b.Again
|
||||
}
|
||||
if err := emitEvent(ctx, js, cred, delivery.VersionNumberedEvent, id, b); err != nil {
|
||||
return fmt.Errorf("saying %s for %s@%s: %w", delivery.VersionNumberedEvent, b.Name, b.Version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// emitEvent says one of the builder's events, mesh.mod.<module>.event.<event>, with the SDK's envelope headers and id as
|
||||
// the bus's dedup id.
|
||||
func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
// The events on the builder's consumer of EVENTS that wake a floor (design 51 *The floor*, DLV-B3b).
|
||||
var (
|
||||
manifestAcceptedSubject = "mesh.mod." + delivery.IndexerModule + ".event." + delivery.ManifestAcceptedEvent
|
||||
conditionClearedSubject = "mesh.mod." + health.KeeperModule + ".event." + health.EventCleared
|
||||
)
|
||||
|
||||
// floorWake asks the floor of the names an event wakes again, by looking at every module again: a trunk build is
|
||||
// numbered before its claim (build.Builder.number).
|
||||
type floorWake struct {
|
||||
b *build.Builder
|
||||
waits *versions.FloorWaits
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// heard takes a message of the consumer in when it is one that wakes a floor, acknowledged, and answers whether it was.
|
||||
func (f *floorWake) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
switch m.Subject() {
|
||||
case manifestAcceptedSubject:
|
||||
f.ack(m)
|
||||
f.wake(ctx, versions.WakeAccepted, "", "")
|
||||
case conditionClearedSubject:
|
||||
f.ack(m)
|
||||
var e health.Event
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
f.logf("%s holds what is no condition, left: %v", m.Subject(), err)
|
||||
return true
|
||||
}
|
||||
for _, module := range conditionModules(e.Subject.ID) {
|
||||
f.wake(ctx, versions.WakeCleared, "", module)
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *floorWake) ack(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
f.logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// wake looks at every module again when the event wakes a name, and says which.
|
||||
func (f *floorWake) wake(ctx context.Context, kind, seat, module string) {
|
||||
names := f.waits.Wake(kind, seat, module, "")
|
||||
if len(names) == 0 {
|
||||
return
|
||||
}
|
||||
f.logf("the floor of %s is asked again: %s", strings.Join(names, ", "), kind)
|
||||
f.b.Reconsider(ctx)
|
||||
}
|
||||
|
||||
// conditionModules are the tokens of a condition's subject id (`<node>/<module>`, `<module>.<check>`): a module named
|
||||
// among them is the one whose condition cleared.
|
||||
func conditionModules(id string) []string {
|
||||
return strings.FieldsFunc(id, func(r rune) bool { return r == '/' || r == '.' || r == ':' })
|
||||
}
|
||||
|
||||
// doneVia says whether a build key is done, read from the builder's own builds state: the build's file is held.
|
||||
func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (bool, error) {
|
||||
return func(ctx context.Context, module, fp string) (bool, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return cur.State == delivery.BuildDone, nil
|
||||
}
|
||||
}
|
||||
|
||||
// removeVia calls a registry seat's remove for one tagged version, as the builder's own account (design 51 *Tagged
|
||||
// versions*): the reason tag-gone; the seat says removed on its events, or answers kept when another tag names the
|
||||
// digest (the registry part's RemoveAnswer: kept, reason, tags), which ends the remove.
|
||||
func removeVia(call toolCaller) func(ctx context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) {
|
||||
return func(_ context.Context, seat string, e versions.Entry) (versions.RemoveOutcome, error) {
|
||||
raw, err := call("seat:"+seat+".remove", map[string]any{"name": e.Name, "version": e.Version, "tag": e.Tag, "reason": "tag-gone"})
|
||||
if err != nil {
|
||||
return versions.RemoveOutcome{}, err
|
||||
}
|
||||
var a ociartifactregistry.RemoveAnswer // the four registry seats share the part's answer
|
||||
if len(raw) > 0 && json.Unmarshal(raw, &a) != nil {
|
||||
return versions.RemoveOutcome{}, fmt.Errorf("%s's remove answered what is no answer of it: %s", seat, raw)
|
||||
}
|
||||
return versions.RemoveOutcome{Kept: a.Kept, Reason: a.Reason, Tags: a.Tags}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// tagAsked answers an unanswered value of tag-asks by compare-and-set at its revision, said in the log.
|
||||
func tagAsked(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
t, err := versions.DecodeTagAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a tag ask, left: %v", versions.TagAsksState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !t.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
t = acts.TagAsked(ctx, t, time.Now())
|
||||
logf("%s: %s of %s %s by %s (%s): %s", e.Key(), map[bool]string{true: "untag", false: "tag ask"}[t.Untag], t.Repository, t.Branch, t.By, t.Why, t.Answer)
|
||||
data, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", versions.TagAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -3,44 +3,20 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261104174638-125929dbc459
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
|
||||
|
||||
@@ -1,37 +1,17 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261104174638-125929dbc459 h1:DA4SFG/gKyG+aPJbfUpAUad2VeITIiKjJ6qOPmDhSA8=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261104174638-125929dbc459/go.mod h1:zR1HsAKZVbQsBIY5wPyrqI2WHInK268tITzGl2oqHro=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-20261104174344-ae5266e77639 h1:I6uVDO9E+i3D9/n6KQ2RWYRs3kXYCqueiKXynifQo8w=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:ER2NVrd81uJo1lN0YCGA7OBxL910IHRgWYtxpsnHGF0=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-20261104174344-ae5266e77639 h1:Rs5jOa7EWlpKuaH6+ExyGLN9k9UUi8N7upP9Ac3k+GY=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:fNfjFkP1+suYb12HJkapQe5W3cF98JsTcLBI/syoLHc=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-20261104174344-ae5266e77639 h1:VG+sZltyBIKrd9cwFJyKkInz04SxWBxUlJWfAuo6774=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:Nq7glLZSZ8zOLTh1pXDnKZi+MxsMRyHw/bZODC3mvrY=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-20261104174344-ae5266e77639 h1:ROS9UG/Dg0NpsaDuEmDFV9CsIC1R92sOm3QauF7IvL0=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:/Pke1v2VnSFcR8ic+fo/kNCg3JFYK6ILTal72Ca9520=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639 h1:5HeZzk6tKzcSwVivD3IcIvQ/2WIvysZvNvNKyq/oIBA=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:pPxwhTMb3hIk7UUrBicJnGQCqfTEoBGJvNFspEK2U18=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-20261104174344-ae5266e77639 h1:oAXU1t8A1rGIr82JPBaNYSGLicXqhWglKaEAtPmrD0c=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:Y5kgBh0hBbL/hgc+abCF9Vbhov+vCsB0ckpRzUw1D+0=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639 h1:YQ6eJvc+RYUVIVf+oX+R4TC1dYL8TLLMZ0/JPDU1tp4=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639/go.mod h1:TB20TsQuOd5aer+cXL6PbaYyg2RvgL1R4MzD61Oks3A=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-20261104174344-ae5266e77639 h1:ILh7uctNwgx/89qHoR/H2kz5TpWeKEeIH0O2pDPE7Iw=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:JRtTjIjMTkzn59IN4vCYei9ZwJGFnN1Puwame4V/sD0=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639 h1:+bkN8op2yrAlBYrOB+hYnIoIZkbjXsNmc/4//uMnL4E=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-20261104174344-ae5266e77639/go.mod h1:ooQa2CiP8JOvhP0bdQvwurV5wskRE7Cm68ecURBbCwU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -45,5 +25,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
|
||||
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
|
||||
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
|
||||
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
|
||||
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
|
||||
|
||||
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
|
||||
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
|
||||
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch cur.State {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
|
||||
default:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
|
||||
}
|
||||
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
|
||||
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
|
||||
}
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
|
||||
}
|
||||
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
|
||||
return BuildAgainResult{Revision: nrev}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
key *delivery.BuilderBuild
|
||||
fp string
|
||||
refuse string
|
||||
}{
|
||||
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
|
||||
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
|
||||
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
|
||||
{name: "a key with no build is refused", refuse: "has no build key"},
|
||||
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var rev uint64
|
||||
if c.key != nil {
|
||||
c.key.Manifest = manifestD
|
||||
rev = putBuild(t, r.st, r.key, *c.key)
|
||||
}
|
||||
before := bytes.Clone(r.st.vals[r.key])
|
||||
fp := fp1
|
||||
if c.fp != "" {
|
||||
fp = c.fp
|
||||
}
|
||||
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
|
||||
if c.refuse != "" {
|
||||
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
|
||||
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,12 +6,10 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -22,8 +20,7 @@ import (
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
|
||||
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
@@ -178,42 +175,6 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
|
||||
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
|
||||
// ask again, so a lone node off the agreed toolchain cannot spin on it.
|
||||
const (
|
||||
DefaultDeclineBound = 30 * time.Minute
|
||||
DefaultDeclinePause = 5 * time.Second
|
||||
)
|
||||
|
||||
func (b *Builder) declineBound() time.Duration {
|
||||
if b.DeclineBound > 0 {
|
||||
return b.DeclineBound
|
||||
}
|
||||
return DefaultDeclineBound
|
||||
}
|
||||
|
||||
func (b *Builder) declinePause() time.Duration {
|
||||
if b.DeclinePause != 0 {
|
||||
return max(b.DeclinePause, 0)
|
||||
}
|
||||
return DefaultDeclinePause
|
||||
}
|
||||
|
||||
// declinedWords names the nodes that declined an ask and why, in the order of their names.
|
||||
func declinedWords(by map[string]string) string {
|
||||
nodes := make([]string, 0, len(by))
|
||||
for n := range by {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var parts []string
|
||||
for _, n := range nodes {
|
||||
parts = append(parts, n+": "+by[n])
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
@@ -292,8 +253,6 @@ func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
|
||||
errDeclined = errors.New("declined")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
@@ -307,42 +266,11 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
var firstDecline time.Time
|
||||
declines := map[string]string{} // node -> why it declined, the last
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errDeclined):
|
||||
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
|
||||
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
|
||||
// checker's decline loop is, issue 890).
|
||||
now := b.now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
declines[step.Node] = why
|
||||
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
|
||||
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
|
||||
}
|
||||
// The pause is a context's bound, not a timer (ADR 0437 §8).
|
||||
pause, stop := context.WithTimeout(ctx, b.declinePause())
|
||||
<-pause.Done()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return step, ask, ctx.Err()
|
||||
}
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
@@ -387,16 +315,6 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
|
||||
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
|
||||
// after a pause, within DeclineBound.
|
||||
return step, step.Reason, errDeclined
|
||||
}
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
return step, step.Reason, errAgain
|
||||
}
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The build runner a build node runs an ask with is mesh-build-go's (novox/hq ADR 0593 §1), another repository's,
|
||||
// which the builder does not import. These tests run an ask as the builder needs it run: fetch once at the job's
|
||||
// fetch arguments with its package versions, build or copy each artifact, answer each one's digest and reference. They
|
||||
// test the builder, not the runner.
|
||||
|
||||
// stepFunc is one step of a build module, as its tool answers it.
|
||||
type stepFunc func(ctx context.Context, args map[string]any) (any, error)
|
||||
|
||||
// askSteps are a build module's steps an ask is run with.
|
||||
type askSteps struct{ Fetch, Build, Copy stepFunc }
|
||||
|
||||
// runAsk runs a build ask with steps and answers its delivery.BuildAnswer, or the error that fails the step.
|
||||
func runAsk(ctx context.Context, steps askSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build == nil {
|
||||
return nil, fmt.Errorf("the ask %s holds no build job", ask.ID)
|
||||
}
|
||||
job := *ask.Build
|
||||
fetch := job.Fetch
|
||||
if len(ask.Packages) > 0 && fetch != nil {
|
||||
fetch = maps.Clone(fetch)
|
||||
fetch["packages"] = maps.Clone(ask.Packages)
|
||||
}
|
||||
answer := delivery.BuildAnswer{}
|
||||
workspace := ""
|
||||
for _, a := range job.Artifacts {
|
||||
var out any
|
||||
var err error
|
||||
if a.From != "" {
|
||||
if steps.Copy == nil {
|
||||
return nil, fmt.Errorf("this build module copies no artifact (%s)", a.Artifact)
|
||||
}
|
||||
out, err = steps.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
|
||||
} else {
|
||||
if workspace == "" {
|
||||
raw, err := asRaw(steps.Fetch(ctx, fetch))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Merge string `json:"merge"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
||||
return nil, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
||||
}
|
||||
if _, candidate := fetch["base"]; candidate && (len(f.Conflicts) > 0 || f.Merge == "") {
|
||||
return nil, fmt.Errorf("the head does not merge into the trunk: %s", strings.Join(f.Conflicts, ", "))
|
||||
}
|
||||
workspace = f.Workspace
|
||||
}
|
||||
args := map[string]any{"workspace": workspace, "module": job.Path, "artifact": a.Artifact}
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
out, err = steps.Build(ctx, args)
|
||||
}
|
||||
raw, err := asRaw(out, err)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, raw)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func asRaw(v any, err error) (json.RawMessage, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
|
||||
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
|
||||
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
|
||||
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
|
||||
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
|
||||
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
|
||||
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
|
||||
|
||||
// BuildAgainResult is the builder's answer to a request to build a key again.
|
||||
type BuildAgainResult struct {
|
||||
// Refused is why nothing was done, "" when the key was written building as a new revision.
|
||||
Refused string
|
||||
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
|
||||
Revision uint64
|
||||
Asks map[string]string
|
||||
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
|
||||
Pending string
|
||||
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
|
||||
// toolchain not agreed): the new revision was written, and nothing was asked.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// handWatch follows one request to build a key again until its asks are put.
|
||||
type handWatch struct {
|
||||
rev uint64
|
||||
lanes int
|
||||
asks map[string]string
|
||||
done chan struct{}
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
func (w *handWatch) settle() {
|
||||
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
|
||||
w.shut = true
|
||||
close(w.done)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) hand(key string, f func(w *handWatch)) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if w := b.hands[key]; w != nil {
|
||||
f(w)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) handClaimed(key string, rev uint64) {
|
||||
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handLanes(key string, n int) {
|
||||
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
|
||||
}
|
||||
|
||||
func (b *Builder) handAsk(key, language, id string) {
|
||||
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
|
||||
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
|
||||
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
|
||||
// ctx is the builder's own context: the build it starts lives on it.
|
||||
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
|
||||
b.init()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: err.Error()}
|
||||
}
|
||||
value, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
|
||||
}
|
||||
if !found {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
|
||||
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
|
||||
case cur.State != delivery.BuildFailed:
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
|
||||
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
|
||||
}
|
||||
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
|
||||
// and the watch and the look are registered before anything else can take the key.
|
||||
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
|
||||
b.mu.Lock()
|
||||
m, known := b.known[module]
|
||||
switch {
|
||||
case !known || m.FP2 != fp:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
|
||||
case m.Manifest == "":
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
|
||||
case b.inflight[key] || b.hands[key] != nil:
|
||||
b.mu.Unlock()
|
||||
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
|
||||
}
|
||||
b.hands[key] = w
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
defer func() {
|
||||
b.mu.Lock()
|
||||
if b.hands[key] == w {
|
||||
delete(b.hands, key)
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
|
||||
b.startOnce(ctx, key, module, m)
|
||||
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
|
||||
defer stop()
|
||||
select {
|
||||
case <-w.done:
|
||||
case <-wait.Done():
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
// The look ended before every ask was put: say how the key stands, not that it was asked.
|
||||
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
|
||||
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
|
||||
switch cur.State {
|
||||
case delivery.BuildFailed:
|
||||
res.Failed = cur.Error
|
||||
case delivery.BuildDone:
|
||||
default:
|
||||
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
|
||||
}
|
||||
}
|
||||
}
|
||||
case !shut:
|
||||
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
|
||||
@@ -68,7 +68,7 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
continue
|
||||
}
|
||||
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
|
||||
Toolchains: b.agreedFor(ctx, cm.Manifest)}
|
||||
if len(cm.Paths) > 0 {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
|
||||
@@ -61,8 +61,8 @@ type ClaimOptions struct {
|
||||
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
|
||||
// clause 6). Nil means none is.
|
||||
Published func(ref string) bool
|
||||
// Stored answers which of the references given the artifact store no longer holds (the seat mesh-artifact-sweep's verb held,
|
||||
// novox/hq ADR 0571 §2, issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// Stored answers which of the references given the artifact store no longer holds (the controller's verb held,
|
||||
// mesh-sdk delivery.HeldVerb; novox/hq issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
|
||||
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
|
||||
@@ -114,11 +114,12 @@ func TestClaimCreatesTheKeyBuildingAndASecondBuilderGetsNothing(t *testing.T) {
|
||||
if second, err := b.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}); err != nil || second != nil {
|
||||
t.Errorf("second builder's claim = %v, %v, want none: the same source is never built twice", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A claim losing the create race gets nothing (merged into this test for the test budget, design 51).
|
||||
st = newMem()
|
||||
func TestClaimLosingTheCreateRaceGetsNothing(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
st.before = func() { seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.b"}) }
|
||||
got, err = Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
if err != nil || got != nil {
|
||||
t.Errorf("claim = %v, %v, want none: the create failed because the key exists", got, err)
|
||||
}
|
||||
@@ -196,11 +197,12 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing
|
||||
if st.writes != writes {
|
||||
t.Errorf("the same value was written again (%d writes, was %d)", st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
// Finish after the key changed is a conflict, not an overwrite (merged into this test for the test budget).
|
||||
st = newMem()
|
||||
c = Claimer{Store: st, By: "novox.a"}
|
||||
got, _ = c.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
func TestFinishAfterTheKeyChangedIsAConflictNotAnOverwrite(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
c := Claimer{Store: st, By: "novox.a"}
|
||||
got, _ := c.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
st.put(got.Key, []byte(`{"v":1,"state":"failed","error":"someone else"}`))
|
||||
err := c.Finish(context.Background(), got, delivery.BuilderBuild{State: delivery.BuildFailed, Error: "mine"})
|
||||
if !errors.Is(err, ErrConflict) {
|
||||
|
||||
@@ -29,7 +29,7 @@ func captured(t *testing.T, name string) json.RawMessage {
|
||||
return step.Result
|
||||
}
|
||||
|
||||
// The captured image steps, run as a pulled ask runs them (runAsk), answer what the builder reads: the trunk's
|
||||
// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's
|
||||
// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a).
|
||||
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
|
||||
for _, tc := range []struct {
|
||||
@@ -37,7 +37,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
publish bool
|
||||
ref bool
|
||||
}{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} {
|
||||
steps := askSteps{
|
||||
steps := buildstep.AskSteps{
|
||||
Fetch: func(context.Context, map[string]any) (any, error) {
|
||||
return json.RawMessage(captured(t, "fetch.json")), nil
|
||||
},
|
||||
@@ -47,7 +47,7 @@ func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // revi
|
||||
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish,
|
||||
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}}
|
||||
out, err := runAsk(context.Background(), steps, ask)
|
||||
out, err := buildstep.RunAsk(context.Background(), steps, ask)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.build, err)
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0593 §2 (F3): the imports a Go build module answers at a build's end reach the done key and the log's end entry,
|
||||
// where the indexer keeps them on the module.
|
||||
func importsABuildAnswersReachTheDoneKeyAndTheLog(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
imports := []string{"git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-sdk/go/announce"}
|
||||
r.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
step := r.answer(a)
|
||||
var answer map[string]any
|
||||
if err := json.Unmarshal(step.Result, &answer); err == nil && a.Language == "go" {
|
||||
answer["imports"] = imports
|
||||
step.Result, _ = json.Marshal(answer)
|
||||
}
|
||||
return step
|
||||
}
|
||||
var logged []delivery.BuildLogEntry
|
||||
var mu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() }
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || strings.Join(v.Imports, ",") != strings.Join(imports, ",") {
|
||||
t.Fatalf("done key %+v, want the imports", v)
|
||||
}
|
||||
if len(logged) != 2 || strings.Join(logged[1].Imports, ",") != strings.Join(imports, ",") || len(logged[0].Imports) != 0 {
|
||||
t.Fatalf("log %+v, want the imports on the end alone", logged)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 26 (the lead's ruling on A2i's finding 2): the out-of-step files a contracts module's build module answers, in a
|
||||
// failed step's result too, reach the log's end entry, sorted and each once, where the indexer raises its finding; a
|
||||
// build that answers none names none.
|
||||
func outOfStepFilesABuildAnswersReachTheLog(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
fail bool
|
||||
files []string
|
||||
want string
|
||||
}{
|
||||
{"a passed build naming files", false, []string{"go/b.go", "go/a.go", "go/b.go"}, "go/a.go,go/b.go"},
|
||||
{"a failed build naming files", true, []string{"ts/x.ts"}, "ts/x.ts"},
|
||||
{"a build naming none", false, nil, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
step := r.answer(a)
|
||||
answer := map[string]any{}
|
||||
_ = json.Unmarshal(step.Result, &answer)
|
||||
if row.files != nil {
|
||||
answer["out-of-step"] = row.files
|
||||
}
|
||||
step.Result, _ = json.Marshal(answer)
|
||||
if row.fail {
|
||||
step.State, step.Reason = delivery.StepFailed, "the generated files are out of step"
|
||||
}
|
||||
return step
|
||||
}
|
||||
var logged []delivery.BuildLogEntry
|
||||
var mu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { mu.Lock(); logged = append(logged, e); mu.Unlock() }
|
||||
r.consider(t, "modules/m")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(logged) != 2 || strings.Join(logged[1].OutOfStep, ",") != row.want || len(logged[0].OutOfStep) != 0 {
|
||||
t.Fatalf("log %+v, want the end naming %q", logged, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Numbers numbers a trunk build at its claim (novox/hq design 51 *The versions*, ADR 0606, ADR 0617 §1) and records the
|
||||
// digests it was built with. Nil numbers nothing.
|
||||
type Numbers interface {
|
||||
// tag is a check's build's tag (TagOf), empty for a trunk build; done the digests of its key when it is done already
|
||||
// (versions.BuildClaim.Done), nil otherwise.
|
||||
Number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string, done map[string]string) (versions.Numbering, error)
|
||||
Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error
|
||||
}
|
||||
|
||||
// VersionNumbers is Numbers over the builder's state versions.
|
||||
type VersionNumbers struct {
|
||||
N *versions.Numberer
|
||||
// TagNumbered is told the versions a check's build of a tag ask's head was numbered (the ask is then marked numbered
|
||||
// in tag-asks, design 51 *Tagged versions*); nil tells nobody.
|
||||
TagNumbered func(ctx context.Context, c versions.BuildClaim, got versions.Numbering)
|
||||
}
|
||||
|
||||
// ClaimOf is the claim a module value's trunk build is numbered by: a package per source revision (Src), the rest per
|
||||
// fingerprint, in merge order (Src).
|
||||
func ClaimOf(module string, m delivery.Module, plan Plan) versions.BuildClaim {
|
||||
c := versions.BuildClaim{Module: module, Path: m.Path, Repository: delivery.RepositoryName(m.Repository), Fingerprint: m.FP2,
|
||||
Commit: m.Commit, Merge: m.Src}
|
||||
if m.Src != 0 {
|
||||
c.Revision = strconv.FormatUint(m.Src, 10)
|
||||
}
|
||||
for _, t := range plan.Tasks {
|
||||
c.Artifacts = append(c.Artifacts, versions.ClaimArtifact{Artifact: t.Artifact, Kind: t.Kind, Language: t.Language})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (v VersionNumbers) Number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string, done map[string]string) (versions.Numbering, error) {
|
||||
c := ClaimOf(module, m, plan)
|
||||
c.Done = done
|
||||
if tag != "" {
|
||||
// A check's build of a head a tag ask names: its packages tagged, below the next version (design 51 *Tagged versions*).
|
||||
c.Tag, c.Head, c.Revision, c.Merge = tag, m.Commit, "", 0
|
||||
}
|
||||
got, err := v.N.Number(ctx, c)
|
||||
if err == nil && tag != "" && len(got.Versions) > 0 && v.TagNumbered != nil {
|
||||
v.TagNumbered(ctx, c, got)
|
||||
}
|
||||
return got, err
|
||||
}
|
||||
|
||||
func (v VersionNumbers) Built(ctx context.Context, module string, m delivery.Module, plan Plan, got versions.Numbering, made []delivery.BuiltArtifact) error {
|
||||
digests := map[string]string{}
|
||||
for _, a := range made {
|
||||
digests[a.Name] = a.Digest
|
||||
}
|
||||
return v.N.Built(ctx, ClaimOf(module, m, plan), got, digests)
|
||||
}
|
||||
|
||||
// TagOf answers the cleaned tag a kept tag ask names for a repository's head (the builder's state tag-asks, design 51
|
||||
// *Tagged versions*): a check's build of that head numbers its packages tagged at its claim.
|
||||
type TagOf func(ctx context.Context, repository, head string) (tag string, ok bool)
|
||||
|
||||
// number numbers a build before its claim, a trunk build's artifacts or a check's build's packages under its tag: false
|
||||
// when it waits for its floor (a package, design 51 *The floor*), and then nothing is claimed; the floor's wake looks at
|
||||
// the module again.
|
||||
func (b *Builder) number(ctx context.Context, module string, m delivery.Module, plan Plan, tag string) (versions.Numbering, bool) {
|
||||
if b.Numbers == nil {
|
||||
return versions.Numbering{}, true
|
||||
}
|
||||
var done map[string]string
|
||||
if tag == "" {
|
||||
done = b.doneDigests(ctx, module, m.FP2)
|
||||
}
|
||||
got, err := b.Numbers.Number(ctx, module, m, plan, tag, done)
|
||||
for _, s := range got.Said {
|
||||
b.logf("%s at %s: %s", module, m.FP2, s.Line)
|
||||
// And in the build log (design 51 *The versions*: it says a version was kept, and why), between the build's start and end.
|
||||
// A look at a key already done builds nothing and has no start or end: a version kept there says nothing new, and is
|
||||
// not logged (each later look, a first view answered or a name heard, would add one).
|
||||
if b.Log != nil && !(done != nil && s.Why == versions.SaidKept) {
|
||||
b.Log(delivery.BuildLogEntry{Event: delivery.BuildSaidEvent, Module: module, FP: m.FP2, Commit: m.Commit, Node: b.Node,
|
||||
At: time.Now().UTC(), Line: s.Line, Why: s.Why})
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// Said, and built as before: an unnumbered build is published by its digest (ADR 0617 §1), numbered at its next look.
|
||||
b.logf("%s at %s: %v", module, m.FP2, fmt.Errorf("not numbered, the versions cannot be used: %w", err))
|
||||
return versions.Numbering{}, true
|
||||
}
|
||||
if got.Wait != "" {
|
||||
b.logf("%s at %s is not built yet: %s", module, m.FP2, got.Wait)
|
||||
return got, false
|
||||
}
|
||||
return got, true
|
||||
}
|
||||
|
||||
// doneDigests are the digests of a trunk build's key by artifact when the key is done, nil otherwise or when it cannot be
|
||||
// read: a look at a done build numbers what it left unnumbered with them (versions.BuildClaim.Done).
|
||||
func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string]string {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil || b.Claimer.Store == nil {
|
||||
return nil
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return nil
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || cur.State != delivery.BuildDone {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, a := range cur.Artifacts {
|
||||
if a.Digest != "" {
|
||||
out[a.Name] = a.Digest
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
func (b *Builder) PackageNamed(ctx context.Context, module string) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
m, ok := b.known[module]
|
||||
b.mu.Unlock()
|
||||
if ok {
|
||||
b.consider(ctx, module, m)
|
||||
}
|
||||
}
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
|
||||
|
||||
@@ -3,12 +3,11 @@ package build
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
|
||||
@@ -305,9 +305,10 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test
|
||||
r := newRig(t, tsPackage)
|
||||
r.bm.packages["lib"] = "@novox/mesh-sdk@0.1.15"
|
||||
paths := map[string]string{"src": hex40('1'), "package.json": hex40('2')}
|
||||
// The named tree the indexer computes from those paths (its own, ADR 0593 §2), given as data: the builder reads it,
|
||||
// never computes it.
|
||||
const tree = delivery.NamedTreePrefix + "5b5e0f3b6f3c1f2a9d8e7c6b5a4938271605f4e3d2c1b0a9f8e7d6c5b4a39281"
|
||||
tree, err := delivery.NamedTree(manifestD, paths)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-sdk", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP2: fp1, Manifest: manifestD, Path: ".",
|
||||
Toolchains: agreedToolchains, Inputs2: &delivery.KeptInputs2{Tree: tree, Paths: paths}})
|
||||
if err != nil {
|
||||
|
||||
@@ -2,10 +2,9 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -6,20 +6,15 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/forms"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
||||
@@ -55,7 +50,7 @@ type Builder struct {
|
||||
// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
|
||||
// rests on it. Nil tells nobody.
|
||||
Answered func(buildModule string)
|
||||
// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
@@ -71,27 +66,8 @@ type Builder struct {
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
|
||||
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
|
||||
DeclineBound, DeclinePause time.Duration
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
|
||||
Log func(delivery.BuildLogEntry)
|
||||
// Now is the builder's clock: of the log, and of the decline bound (DeclineBound runs from the first decline); nil is
|
||||
// time.Now. Node is the node the builder runs on, which asked the build.
|
||||
Now func() time.Time
|
||||
Node string
|
||||
// Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil
|
||||
// numbers nothing.
|
||||
Numbers Numbers
|
||||
// Tags answers a check's build's tag from the kept tag asks; nil numbers no check's build.
|
||||
Tags TagOf
|
||||
// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
|
||||
// answered when last asked, and counted, every node the placement names, present or away, a node with no statement
|
||||
// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
|
||||
// its single reference, and its needs. Nil: no node states anything, and the first forms are kept.
|
||||
Forms func() (present, counted map[string][]string)
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -103,10 +79,8 @@ type Builder struct {
|
||||
plans map[string]planned // by manifest digest
|
||||
cands map[string]delivery.Candidate // by candidates key (candidates.go)
|
||||
via map[string]via // by build key: a candidate's fetch (candidates.go)
|
||||
// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
|
||||
outOfStep map[string][]string
|
||||
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
|
||||
tried map[string]bool // by build key: looked at since this process started
|
||||
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
|
||||
tried map[string]bool // by build key: looked at since this process started
|
||||
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
||||
// waiting for one is built once it is here (ADR 0500 clause 6).
|
||||
published map[string]bool
|
||||
@@ -115,8 +89,6 @@ type Builder struct {
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -134,7 +106,6 @@ func (b *Builder) init() {
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -277,8 +248,8 @@ func (b *Builder) storedOnce(key string) func(ctx context.Context, references []
|
||||
}
|
||||
}
|
||||
|
||||
// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
|
||||
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
|
||||
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
|
||||
// builder: what is built rests on the store's answer, never on the event alone.
|
||||
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
|
||||
@@ -401,61 +372,25 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
|
||||
}()
|
||||
}
|
||||
|
||||
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
|
||||
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
|
||||
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
defer b.handEnded(key)
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
if planErr == nil && !nothing {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
b.tried[key] = true
|
||||
b.mu.Unlock()
|
||||
var numbering versions.Numbering
|
||||
tag := ""
|
||||
if !trunk && b.Tags != nil && b.refOf(module, m.FP2) != "" {
|
||||
tag, _ = b.Tags(ctx, delivery.RepositoryName(m.Repository), m.Commit)
|
||||
}
|
||||
if (trunk || tag != "") && planErr == nil && !nothing {
|
||||
var goOn bool
|
||||
if numbering, goOn = b.number(ctx, module, m, plan, tag); !goOn {
|
||||
return
|
||||
}
|
||||
}
|
||||
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
|
||||
Stored: b.storedOnce(key)})
|
||||
if err != nil {
|
||||
@@ -465,8 +400,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now()))
|
||||
if cl.Unasked != nil {
|
||||
b.logf("%v", cl.Unasked)
|
||||
}
|
||||
@@ -516,14 +449,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
b.logf("%s: the build could not be written: %v", cl.Key, err)
|
||||
return
|
||||
}
|
||||
ended := LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result)
|
||||
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
|
||||
b.logBuild(ended)
|
||||
if result.State == delivery.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
|
||||
if err := b.Numbers.Built(ctx, module, m, plan, numbering, result.Artifacts); err != nil {
|
||||
b.logf("%s: %v", cl.Key, err)
|
||||
}
|
||||
}
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
}
|
||||
@@ -544,11 +469,10 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := buildrun.LanguageOf[bm]
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
@@ -568,8 +492,6 @@ type lane struct {
|
||||
toolchain *delivery.ToolchainUsed
|
||||
digests map[string]string // artifact -> digest
|
||||
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
|
||||
imports []string // the mesh packages the lane's programs import, as its build module answered them
|
||||
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -584,8 +506,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
for _, bm := range plan.Modules() {
|
||||
if prior != nil && len(only) > 0 && !only[bm] {
|
||||
kept = append(kept, bm)
|
||||
@@ -595,12 +516,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||
language := buildrun.LanguageOf[bm]
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
sitting = append(sitting, SittingOutWords(language, ag)...)
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
@@ -616,18 +535,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||
}
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.handLanes(key, len(lanes))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module, asked) }()
|
||||
}
|
||||
wg.Wait()
|
||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||
b.keepOutOfStep(key, lanes)
|
||||
}
|
||||
var errs []string
|
||||
for _, l := range lanes {
|
||||
if l.err != nil {
|
||||
@@ -643,10 +556,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
return failed("%s", keyError(errs))
|
||||
}
|
||||
var artifacts []delivery.BuiltArtifact
|
||||
var present, counted map[string][]string
|
||||
if b.Forms != nil {
|
||||
present, counted = b.Forms()
|
||||
}
|
||||
keptModule := map[string]bool{}
|
||||
for _, bm := range kept {
|
||||
keptModule[bm] = true
|
||||
@@ -675,7 +584,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
return failed("%s", err)
|
||||
}
|
||||
built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d}
|
||||
var answered []string
|
||||
for _, l := range lanes {
|
||||
if l.module != t.Module {
|
||||
continue
|
||||
@@ -687,11 +595,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node)
|
||||
}
|
||||
if r != "" {
|
||||
answered = append(answered, r)
|
||||
built.Reference = r
|
||||
}
|
||||
}
|
||||
// The forms a counted node's engine reads are kept, the oldest in the single reference (ADR 0624 §1, §3).
|
||||
artifacts = append(artifacts, forms.Current.Choose(built, answered, present, counted))
|
||||
artifacts = append(artifacts, built)
|
||||
}
|
||||
var steps []string
|
||||
for _, l := range lanes {
|
||||
@@ -706,14 +613,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
}
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(sitting) > 0 {
|
||||
step += " (" + strings.Join(sitting, "; ") + ")"
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
|
||||
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1)
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used}
|
||||
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
|
||||
// A reference the build module answered that does not name the artifact and its digest is no reference.
|
||||
return failed("%s%v", delivery.NotStored, err)
|
||||
@@ -808,17 +711,13 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
v := n.cl.Build
|
||||
v.State, v.By, v.Step, v.Error, v.Artifacts, v.Needs = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil, nil
|
||||
v.State, v.By, v.Step, v.Error, v.Artifacts = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil
|
||||
v.Asks = maps.Clone(v.Asks)
|
||||
if v.Asks == nil {
|
||||
v.Asks = map[string]string{}
|
||||
}
|
||||
v.Asks[language] = id
|
||||
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
|
||||
return err
|
||||
}
|
||||
n.b.handAsk(n.cl.Key, language, id)
|
||||
return nil
|
||||
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||
}
|
||||
|
||||
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||
@@ -865,11 +764,6 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
job.Fetch = req.Args()
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
|
||||
if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
}
|
||||
if id := asked.resume(language); id != "" {
|
||||
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||
if b.Held != nil {
|
||||
@@ -904,12 +798,6 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
return
|
||||
}
|
||||
l.node = step.Node
|
||||
// The build module's compare of a contracts module's generated files (design 26, CAT-BG5/BN2) answers the files out
|
||||
// of step as delivery.BuildAnswer.OutOfStep, in a failed step's result too: read before the step's state is.
|
||||
var compared delivery.BuildAnswer
|
||||
if len(step.Result) > 0 && json.Unmarshal(step.Result, &compared) == nil {
|
||||
l.outOfStep = compared.OutOfStep
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason)
|
||||
return
|
||||
@@ -919,7 +807,6 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
|
||||
return
|
||||
}
|
||||
l.imports = answer.Imports
|
||||
built := map[string]delivery.AskBuilt{}
|
||||
for _, a := range answer.Artifacts {
|
||||
built[a.Artifact] = a
|
||||
@@ -949,117 +836,3 @@ func needsFetch(tasks []Task) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *Builder) now() time.Time {
|
||||
if b.Now != nil {
|
||||
return b.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// logBuild tells the log an entry, said in the journal when it cannot be one.
|
||||
func (b *Builder) logBuild(e delivery.BuildLogEntry) {
|
||||
if b.Log == nil {
|
||||
return
|
||||
}
|
||||
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
|
||||
b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err)
|
||||
return
|
||||
}
|
||||
b.Log(e)
|
||||
}
|
||||
|
||||
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
|
||||
func (b *Builder) refOf(module, fp string) string {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.via[key].Ref
|
||||
}
|
||||
|
||||
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
|
||||
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
|
||||
// that asked it and the time.
|
||||
func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry {
|
||||
dir := m.Path
|
||||
if dir == "" {
|
||||
dir = delivery.RootPath
|
||||
}
|
||||
return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10),
|
||||
Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()}
|
||||
}
|
||||
|
||||
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
|
||||
// was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build.
|
||||
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
|
||||
e := LogStarted(module, m, ref, by, at)
|
||||
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
|
||||
if result.Manifest != "" {
|
||||
e.Manifest = result.Manifest
|
||||
}
|
||||
if m.Inputs2 != nil {
|
||||
e.Against = append([]string(nil), m.Inputs2.Bases...)
|
||||
}
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
|
||||
return e
|
||||
}
|
||||
|
||||
// keepOutOfStep holds, by build key, the out-of-step files the lanes' build modules answered, sorted and each once, until
|
||||
// the build's end entry takes them (takeOutOfStep); a build with none holds nothing.
|
||||
func (b *Builder) keepOutOfStep(key string, lanes []*lane) {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lanes {
|
||||
for _, f := range l.outOfStep {
|
||||
seen[f] = true
|
||||
}
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if len(seen) == 0 {
|
||||
delete(b.outOfStep, key)
|
||||
return
|
||||
}
|
||||
if b.outOfStep == nil {
|
||||
b.outOfStep = map[string][]string{}
|
||||
}
|
||||
b.outOfStep[key] = slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// takeOutOfStep answers the out-of-step files held for a build key and forgets them.
|
||||
func (b *Builder) takeOutOfStep(key string) []string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
files := b.outOfStep[key]
|
||||
delete(b.outOfStep, key)
|
||||
return files
|
||||
}
|
||||
|
||||
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
|
||||
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
|
||||
// kept one reports only what it built.
|
||||
func importsOf(lanes []*lane) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lanes {
|
||||
for _, p := range l.imports {
|
||||
seen[p] = true
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
return slices.Sorted(maps.Keys(seen))
|
||||
}
|
||||
|
||||
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
|
||||
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
|
||||
func SittingOutWords(language string, a delivery.Agreement) []string {
|
||||
var out []string
|
||||
for _, s := range a.SittingOut {
|
||||
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user