Compare commits

..
Author SHA1 Message Date
jschoubben ab2535158a Pin the group's mesh-sdk head 0d97c09c for the retake class (issue 1748)
mesh/delivery-group group fix/1748-retake-and-resolver checking: 2 of 2 member(s) ready
mesh/delivery closed: the pull request closed unmerged
mesh/check pass ab2535158a45 on ae31b66a9e33 | unit 6409 run, 187 kept, 14 skipped | integration 6544 run, 220 kept, 23 skipped | over bound: unit, in…
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:45:23 +01:00
jschoubben 1d9ac0c8da The backoff's claim alone counts a retake; the guard names armRetake the one bounded wait; 60 s backoff; the prefix only when every lane is retakeable (issue 1748 review 1)
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:41:57 +01:00
jschoubben a3a11a8f55 Fold the retake integration test into the failed-key test as a subtest, so the integration count stays (issue 1748)
Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:33:46 +01:00
jschoubben d134b8dfec The builder takes a retakeable failure again after a backoff, at most three times, the count kept on the key (issue 1748)
A name lookup that timed out passes by itself, but nothing fired for the key after its failure, so it stayed failed until a person built it again.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 22:30:22 +01:00
1405 changed files with 20816 additions and 198309 deletions
-6
View File
@@ -2,9 +2,3 @@ vendor/
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
modules/*/cmd/*/mesh-*
!modules/*/cmd/*/mesh-*.go
# `go build .` at a module's root leaves its program there too (the view's, met a second time).
modules/*/mesh-*
# The delivery SDK's commands (design 47 G §464), built in their own directory or at the SDK's root.
/modules/mesh-delivery-sdk/cmd/seatgen/seatgen
/modules/mesh-delivery-sdk/seatgen/seatgen
/modules/mesh-delivery-sdk/mesh-pack
-293
View File
@@ -1,293 +0,0 @@
// Package asks holds the layout of the assigner's state `asks`: the keys of the asks the assigner puts to a person and the
// value it keeps at each, which the assigner writes and the view and the teller read (novox/hq ADR 0634 §1, ADR 0635 §5). It
// is the assigner's own published package. The gate's merge ask (`merge.<group>.<heads hash>`) needs proof of who answered
// (ADR 0421 §1), so it is answered on the phone alone until issue 953 closes. The assigner's asks need no such proof, and the
// view may answer them: the hold of a fingerprint whose health check failed (`hold.<module>.<fingerprint hex>`: release it or
// keep it held), the restore of a data module set back (`restore.<node>.<module>.<fingerprint hex>`: restore and lose newer
// writes, or leave it stopped, ADR 0429 §6), a backup that failed before an apply, a move a person must allow for a module of
// policy record, and the authorising ask of a proposal of assignments.
package asks
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/kept"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const (
// AskKindHold is the kind of the assigner's ask about a held fingerprint.
AskKindHold = "hold"
// AskKindRestore is the kind of the assigner's ask about a data module set back.
AskKindRestore = "restore"
// AskKindBackup is the kind of the assigner's ask about a backup before an apply that failed: back up again, or go on
// without one (novox/hq ADR 0539 clause 16).
AskKindBackup = "backup"
// AskKindRecord is the kind of the assigner's ask about a move a person must allow, for a module of policy record: a
// composition judged to recreate a container or move data, or one that cannot be judged (ADR 0539 clause 14).
AskKindRecord = "record"
// AskKindAssignments is the kind of the authorising ask about a proposal of assignments (novox/hq ADR 0567 §5):
// `assignments.<proposal id>`. The kind needs no proof, as the bus's put does not: its yes is an Approve, which only a
// channel proving who answered takes and the view never does (ViewAnswerFits).
AskKindAssignments = "assignments"
// AskKindOutside is the kind of the assigner's ask about a moved module whose failing health check has waited past its bound
// on an outside cause (design 47 G §706): go on, or count it as failed. `outside.<node>.<module>.<fingerprint hex>`.
AskKindOutside = "outside"
// AskKindRevert is the kind of the assigner's ask about a held revert, one with no restore point (design 47 G §710): keep it
// held, or start the reverted build on the live data. `revert.<node>.<module>.<fingerprint hex>`, the reverted build's.
AskKindRevert = "revert"
)
// AskKey is an ask's key read whole: its Kind, and the parts its kind has: a merge's Group and heads Hash, a hold's Module
// and fingerprint FP, a restore's, a backup's, an outside's and a revert's Node, Module and FP, a record's Node, Module and the digest Composed of the
// composition it would allow, the assignments' Proposal.
type AskKey struct {
Proposal string
Kind string
Group string
Hash string
Node string
Module string
FP string
Composed string
}
// AskNeedsProof says whether an ask of this kind must prove who answered it, so the view may not answer it: the merge's
// Approve does (ADR 0421 §1, issue 953), the assigner's hold and restore do not. An unknown kind is refused.
func AskNeedsProof(kind string) (bool, error) {
switch kind {
case meshdeliveryclient.AskKindMerge:
return true, nil
case AskKindHold, AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert:
return false, nil
}
return false, fmt.Errorf("asks: %q is not a kind of ask, %s, %s, %s, %s, %s, %s, %s or %s", kind, meshdeliveryclient.AskKindMerge, AskKindHold,
AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert)
}
// HoldAskKey is the key of the ask about a module's held fingerprint: `hold.<module>.<hex>`.
func HoldAskKey(module, fingerprint string) (string, error) {
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindHold + "." + key, nil
}
// RestoreAskKey is the key of the ask about a data module set back on a node from a fingerprint:
// `restore.<node>.<module>.<hex>`.
func RestoreAskKey(node, module, fingerprint string) (string, error) {
if err := keys.CheckName("node", node); err != nil {
return "", err
}
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindRestore + "." + node + "." + key, nil
}
// OutsideAskKey is the key of the ask about a module's failing check on a node that waited past its bound on an outside cause:
// `outside.<node>.<module>.<hex>`.
func OutsideAskKey(node, module, fingerprint string) (string, error) {
key, err := RestoreAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindOutside + strings.TrimPrefix(key, AskKindRestore), nil
}
// OutsideAbout is what the outside ask is about: `<node>.<module>.<hex>` (design 47 G §706).
func OutsideAbout(key string) string { return strings.TrimPrefix(key, AskKindOutside+".") }
// BackupAskKey is the key of the ask about a module's failed backup on a node before it applies a fingerprint:
// `backup.<node>.<module>.<hex>`.
func BackupAskKey(node, module, fingerprint string) (string, error) {
part, err := catalogkey.KeyPart(fingerprint)
if err != nil {
return "", err
}
key, err := nodeengineclient.ComposedKey(node, module, part)
if err != nil {
return "", err
}
return AskKindBackup + "." + key, nil
}
// RevertAskKey is the key of the ask about a held revert of a module on a node to the reverted build's fingerprint:
// `revert.<node>.<module>.<hex>` (design 47 G §710).
func RevertAskKey(node, module, fingerprint string) (string, error) {
key, err := BackupAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindRevert + strings.TrimPrefix(key, AskKindBackup), nil
}
// AssignmentsAskKey is the key of the authorising ask about a proposal of assignments: `assignments.<proposal id>` (ADR 0567 §5).
func AssignmentsAskKey(proposal string) (string, error) {
if err := meshmoduleassignerclient.CheckProposalID(proposal); err != nil {
return "", err
}
return AskKindAssignments + "." + proposal, nil
}
// RecordAskKey is the key of the ask about a move of a module on a node to the composition whose digest it names:
// `record.<node>.<module>.<digest hex>`, so a new revision at the same fingerprint is asked again (ADR 0539 clause 14).
func RecordAskKey(node, module, composed string) (string, error) {
have, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return "", err
}
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return "", err
}
return AskKindRecord + "." + have + "." + strings.TrimPrefix(composed, "sha256:"), nil
}
// ParseAnyAskKey reads an ask key of any kind (a merge, hold, restore, backup, record, assignments, outside or revert ask); it refuses an unknown
// kind and a key whose parts do not fit its kind, naming the key.
func ParseAnyAskKey(key string) (AskKey, error) {
kind, rest, _ := strings.Cut(key, ".")
refused := func(err error) (AskKey, error) {
return AskKey{}, fmt.Errorf("asks: %q is not an ask key: %w", key, err)
}
switch kind {
case meshdeliveryclient.AskKindMerge:
_, group, hash, err := meshdeliveryclient.ParseAskKey(key)
if err != nil {
return AskKey{}, err
}
return AskKey{Kind: kind, Group: group, Hash: hash}, nil
case AskKindHold:
module, fp, err := builds.ParseBuilderBuildKey(rest)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Module: module, FP: fp}, nil
case AskKindRestore:
node, build, ok := strings.Cut(rest, ".")
if !ok {
return refused(fmt.Errorf("a restore ask is restore.<node>.<module>.<fingerprint hex>"))
}
if err := keys.CheckName("node", node); err != nil {
return refused(err)
}
module, fp, err := builds.ParseBuilderBuildKey(build)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindOutside:
k, err := ParseAnyAskKey(AskKindRestore + "." + rest)
if err != nil {
return refused(fmt.Errorf("an outside ask is outside.<node>.<module>.<fingerprint hex>"))
}
k.Kind = kind
return k, nil
case AskKindBackup, AskKindRevert:
node, module, part, isNode, err := nodeengineclient.ParseComposedKey(rest)
if err != nil || isNode {
return refused(fmt.Errorf("a %s ask is %s.<node>.<module>.<fingerprint hex>", kind, kind))
}
fp, err := catalogkey.ParseKeyPart(part)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindAssignments:
if err := meshmoduleassignerclient.CheckProposalID(rest); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Proposal: rest}, nil
case AskKindRecord:
parts := strings.Split(rest, ".")
if len(parts) != 3 {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
node, module, isNode, err := nodeengineclient.ParseHaveKey(parts[0] + "." + parts[1])
if err != nil || isNode {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
composed := "sha256:" + parts[2]
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, Composed: composed}, nil
}
_, err := AskNeedsProof(kind)
return refused(err)
}
// Ask is the value of a key in the assigner's state asks: the router's ask whole, as the assigner composes it, so the teller
// publishes it unchanged. Asking counts the times the assigner wrote the ask at this key, as the gate's ask does: 0 or 1 for
// the first, n for the n-th, and the ask's id is meshdeliveryclient.AskingID(key, Asking), so each asking has its own id and
// an answer to an earlier asking is never taken for a later one. The ask is the operator-channel seat's as its JSON: its id,
// its options and its digest are checked by the assigner and the teller through operator-channel's client (novox/hq ADR
// 0627 §4).
type Ask struct {
V int `json:"v"`
Ask json.RawMessage `json:"ask"`
Asking int `json:"asking,omitempty"`
}
func (a Ask) check(key string) error {
ask, err := ParseAnyAskKey(key)
if err != nil {
return err
}
if proof, _ := AskNeedsProof(ask.Kind); proof {
return fmt.Errorf("asks: the assigner asks no %s ask: it needs proof of who answered", ask.Kind)
}
if a.Asking < 0 {
return fmt.Errorf("asks: the ask's asking %d is negative", a.Asking)
}
var fields map[string]json.RawMessage
if len(bytes.TrimSpace(a.Ask)) == 0 || json.Unmarshal(a.Ask, &fields) != nil || fields == nil {
return fmt.Errorf("asks: the value carries no router's ask")
}
return nil
}
// EncodeAsk is a, the ask at key, as JSON at keys.Version; it refuses a key of a kind that needs proof or of no kind, a
// negative asking, and a value carrying no router's ask.
func EncodeAsk(key string, a Ask) ([]byte, error) {
if err := a.check(key); err != nil {
return nil, err
}
a.V = keys.Version
return json.Marshal(a)
}
// DecodeAsk reads the ask at key; it refuses an unknown version (keys.ErrUnknownVersion) and what EncodeAsk refuses.
func DecodeAsk(key string, data []byte) (Ask, error) {
var a Ask
if err := kept.Decode(data, &a); err != nil {
return Ask{}, err
}
if err := a.check(key); err != nil {
return Ask{}, err
}
return a, nil
}
// SameAsk says whether two asks are the same value: the same version, asking and router's ask, its JSON compared compacted.
func SameAsk(a, b Ask) bool {
return a.V == b.V && a.Asking == b.Asking && bytes.Equal(compacted(a.Ask), compacted(b.Ask))
}
func compacted(raw json.RawMessage) []byte {
var b bytes.Buffer
if json.Compact(&b, raw) != nil {
return raw
}
return b.Bytes()
}
@@ -1,89 +0,0 @@
package asks
import (
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The keys equal what the assigner has always written (the expected strings are the trunk's own output for the same input),
// and each reads back to its parts.
func TestTheAssignersAskKeysKeepTheirFormsAndTheValueItsRules(t *testing.T) {
fp := "src2:" + strings.Repeat("ab", 32)
hex := strings.Repeat("ab", 32)
digest := "sha256:" + strings.Repeat("cd", 32)
hold, _ := HoldAskKey("gitea", fp)
restore, _ := RestoreAskKey("n1", "gitea", fp)
backup, _ := BackupAskKey("n1", "novox.be", fp)
assignments, _ := AssignmentsAskKey("p-20261028t183000-1a2b3c")
record, _ := RecordAskKey("n1", "gitea", digest)
revert, _ := RevertAskKey("n1", "nats", fp)
for key, want := range map[string]struct {
got string
back AskKey
}{
"hold": {hold, AskKey{Kind: "hold", Module: "gitea", FP: fp}},
"restore": {restore, AskKey{Kind: "restore", Node: "n1", Module: "gitea", FP: fp}},
"backup": {backup, AskKey{Kind: "backup", Node: "n1", Module: "novox.be", FP: fp}},
"assignments": {assignments, AskKey{Kind: "assignments", Proposal: "p-20261028t183000-1a2b3c"}},
"record": {record, AskKey{Kind: "record", Node: "n1", Module: "gitea", Composed: digest}},
"revert": {revert, AskKey{Kind: "revert", Node: "n1", Module: "nats", FP: fp}},
} {
back, err := ParseAnyAskKey(want.got)
if err != nil || back != want.back {
t.Errorf("%s: ParseAnyAskKey(%q) = %+v, %v; want %+v", key, want.got, back, err, want.back)
}
}
for got, want := range map[string]string{hold: "hold.gitea." + hex, restore: "restore.n1.gitea." + hex, backup: "backup.n1.novox=2ebe." + hex,
assignments: "assignments.p-20261028t183000-1a2b3c", record: "record.n1.gitea." + strings.Repeat("cd", 32), revert: "revert.n1.nats." + hex} {
if got != want {
t.Errorf("key %q, want %q", got, want)
}
}
merge, err := ParseAnyAskKey("merge.fix/1=2e2.d106c7950d28")
if err != nil || merge.Kind != "merge" || merge.Group != "fix/1.2" || merge.Hash != "d106c7950d28" {
t.Errorf("a merge ask key: %+v, %v", merge, err)
}
for _, bad := range []string{"", "other.x", "hold.gitea", "hold.gitea.src2:" + hex, "restore.n1", "backup.n1.node", "backup.n1.gitea." + hex[:60],
"assignment.AB", "record.n1.gitea." + hex[:10], "record.n1.node." + hex, "merge.g", "revert.n1.node", "revert.n1.nats." + hex[:60]} {
if _, err := ParseAnyAskKey(bad); err == nil {
t.Errorf("ParseAnyAskKey(%q) reads a wrong key", bad)
}
}
if proof, err := AskNeedsProof("merge"); err != nil || !proof {
t.Error("a merge ask needs proof")
}
for _, kind := range []string{"hold", "revert"} {
if proof, err := AskNeedsProof(kind); err != nil || proof {
t.Errorf("a %s ask needs no proof", kind)
}
}
if _, err := AskNeedsProof("view"); err == nil {
t.Error("an unknown kind of ask is taken")
}
// The value: the router's ask whole; an ask of a kind that needs proof, a negative asking and no ask are refused.
good := Ask{Ask: []byte(`{"id":"x"}`), Asking: 2}
data, err := EncodeAsk(hold, good)
if err != nil {
t.Fatalf("EncodeAsk: %v", err)
}
back, err := DecodeAsk(hold, data)
if err != nil || !SameAsk(back, Ask{V: keys.Version, Ask: []byte(` {"id": "x"} `), Asking: 2}) {
t.Errorf("DecodeAsk = %+v, %v", back, err)
}
for name, c := range map[string]struct {
key string
a Ask
}{"a merge ask": {"merge.g.d106c7950d28", good}, "a negative asking": {hold, Ask{Ask: good.Ask, Asking: -1}}, "no ask": {hold, Ask{}},
"a null ask": {hold, Ask{Ask: []byte("null")}}, "an unknown kind": {"other.x", good}} {
if _, err := EncodeAsk(c.key, c.a); err == nil {
t.Errorf("EncodeAsk takes %s", name)
}
}
if _, err := DecodeAsk(hold, []byte(`{"v":2,"ask":{}}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("DecodeAsk of another version = %v", err)
}
}
@@ -1,23 +0,0 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
)
require (
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
)
@@ -1,225 +0,0 @@
package main
import (
"context"
"maps"
"slices"
"strings"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The waits for an acting part's new version in the steps the assigner holds (design 47 G §266, §268, §276): an entry of the assignment
// graph whose acting edge names composing or placing waits, its want held, until the part's version runs on the edge's node. The wait is
// kept in the state `waits` under `acting.<node>.<module>.<part>`, said as `wait-held` when it starts and `wait-ended` (ran, released,
// replaced) when it ends; `assigner_waits` reads it and `assigner_release` with how `now` lets the entry move with the part's running
// version. A node away starts no wait: its clock starts when the node is present (§268).
// actingOf is the Module.Acting of module: the held acting edges of its entry on a node, those whose wait was not released left out.
func (p *passer) actingOf(module string, released map[string]bool) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
if p.engineOn(node) == module {
return nil // the node-engine's own want is held by no acting edge, so it is written first (§268, A-323, A-330)
}
for _, e := range p.entryEdges(node, module) {
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
}
}
return out
}
}
// engineOn is the module that holds the seat node-engine on node in the revision the pass computed, read from the acting edges'
// seat (A-330); "" when none is read.
func (p *passer) engineOn(node string) string {
if p.graph == nil {
return ""
}
return assigngraph.Engines(slices.Collect(maps.Values(p.graph.computed.Entries)))[node]
}
// entryEdges are the acting edges of the entry of module on node in the revision the pass computed.
func (p *passer) entryEdges(node, module string) []seat.Edge {
g := p.graph
if g == nil {
return nil
}
for _, ref := range g.computed.Revision.Entries {
if ref.Node == node && ref.Module == module {
var out []seat.Edge
for _, e := range g.computed.Entries[ref.Digest].Edges {
if e.Kind == "acting" {
out = append(out, e)
}
}
return out
}
}
return nil
}
// actingReleased is the waits released by hand and not yet run, by `<key>@<part's fingerprint>`.
func (p *passer) actingReleased() map[string]bool {
out := map[string]bool{}
for key, raw := range p.w.read(buckets.Waits()) {
if w, err := acting.Decode(raw); err == nil && w.Released && (acting.Is(key) || acting.IsOrder(key)) {
out[key+"@"+w.Fp] = true
}
}
return out
}
// actingWaits keeps the acting waits after the pass's wants: it starts the wait of each entry held for a part's version, ends the
// wait of one that ran, was replaced or was released, and says each.
func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]meshwants.Want) {
g := p.graph
if g == nil || p.kvs[buckets.Waits()] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits())
standing := map[string]bool{}
engines := assigngraph.Engines(slices.Collect(maps.Values(g.computed.Entries)))
held := map[string]bool{} // `<node>/<module>` of each entry held for a part's version
for _, ref := range g.computed.Revision.Entries {
entry := g.computed.Entries[ref.Digest]
if w, ok := wanted[ref.Module][ref.Node]; ok && w.Fp == entry.Fp {
continue // the entry's version is wanted already: nothing moves, so nothing waits
}
if engines[ref.Node] == ref.Module {
continue // the node-engine's own want waits for no acting part (§268, A-323, A-330)
}
for _, e := range entry.Edges {
if e.Kind != "acting" || !acting.Held(e.Step) {
continue
}
key := acting.Key(ref.Node, ref.Module, e.Module)
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
continue
}
standing[key] = true
held[ref.Node+"/"+ref.Module] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == entry.Fp && w.Fp == e.Fp {
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[ref.Node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: e.Step, Node: ref.Node, Module: ref.Module, Part: e.Module, PartNode: e.Node, Fp: e.Fp, Entry: entry.Fp,
Since: now, ClockFrom: now, Bound: acting.BoundOf(e).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.Fp+"."+entry.Fp); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", ref.Module, ref.Node, e.Module, err)
continue
}
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
}
}
p.actingCycles(g.computed.Entries, engines, held, n)
for key, raw := range kept {
if !acting.Is(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
how := acting.HowRan
if w.Released {
how = acting.HowReleased
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module())
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits(), key)
}
}
// actingReleaseSaid says wait-ended of a wait released by hand, once, and keeps its record so the entry is not held again.
func (p *passer) actingReleaseSaid(ctx context.Context, key string, w acting.Wait) {
if !w.Released || w.EndSaid {
return
}
body := acting.Body(w, assignment.Module())
body.How = acting.HowReleased
body.By = w.By
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+acting.HowReleased); err != nil {
p.logf("the release of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
return
}
w.EndSaid = true
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
}
// edgeOf is the edge a kept wait stands on, as an acting edge.
func (p *passer) edgeOf(w acting.Wait) (assign.ActingEdge, bool) {
return assign.ActingEdge{Node: w.PartNode, Part: w.Part, Fp: w.Fp, Step: w.Step}, true
}
// actingCycles writes the finding acting-cycle of each held want whose holding edges lie on a cycle no rule breaks, and clears it of
// every other want, a finding a pass before or another process wrote included (design 47 G A-323, A-330). An edge holds while its
// step is the assigner's, its part's version does not run and its wait was not released; the node-engine's own want is held by none.
func (p *passer) actingCycles(entries map[string]seat.AssignmentEntry, engines map[string]string, held map[string]bool, n assign.Nodes) {
if p.findings.Put == nil {
return
}
released := p.actingReleased()
cycles := assigngraph.HeldCycles(slices.Collect(maps.Values(entries)), func(e seat.AssignmentEntry, edge seat.Edge) bool {
return held[e.Node+"/"+e.Module] && engines[e.Node] != e.Module && edge.Kind == "acting" && acting.Held(edge.Step) &&
!released[acting.Key(e.Node, e.Module, edge.Module)+"@"+edge.Fp] && !n.Runs(assign.ActingEdge{Node: edge.Node, Part: edge.Module, Fp: edge.Fp, Step: edge.Step})
})
raised := map[string]bool{}
for _, at := range slices.Sorted(maps.Keys(cycles)) {
node, module, _ := strings.Cut(at, "/")
o := assign.ActingCycleFinding(node, module, cycles[at])
key := mcseatKey(o)
raised[key] = true
if p.said[key+"="+o.Summary] {
continue // written already as it stands
}
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", key, err)
continue
}
p.said[key+"="+o.Summary] = true
}
if p.findings.Keys == nil || p.findings.Delete == nil {
return
}
keys, err := p.findings.Keys()
if err != nil {
p.logf("the findings %s are not read, so none is cleared this pass: %v", assign.KindActingCycle, err)
return
}
for _, key := range keys {
if strings.HasSuffix(key, "."+assign.KindActingCycle) && !raised[key] {
if err := p.findings.Delete(key); err != nil {
p.logf("the finding %s is not cleared: %v", key, err)
continue
}
for said := range p.said {
if strings.HasPrefix(said, key+"=") {
delete(p.said, said)
}
}
}
}
}
@@ -1,184 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §266, §268, §276 (AS9): an entry with an acting edge in composing waits for the part's version to run on the edge's node;
// the wait is kept, said as wait-held with its bound and clock-from, read by assigner_waits, released by assigner_release with how
// `now` and ended when the version runs.
func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T) {
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, now)
var events []string
var bodies []meshwants.Wait
p.publishWants = func(event string, body any, id string) error {
events = append(events, event)
bodies = append(bodies, body.(meshwants.Wait))
return nil
}
entry := meshwants.AssignmentEntry{Node: "ace", Module: "nats", Fp: fpTest, Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Seat: "node-engine", Step: "compose", Bound: "15m"},
{Kind: "acting", Node: "ace", Module: "indexer", Fp: "src2:idx", Seat: "mesh-catalog", Step: "take-in"}}}
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
ctx := context.Background()
if got := p.actingOf("nats", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
t.Fatalf("the assigner holds composing only: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted) // said once
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("events %v", events)
}
b := bodies[0]
if b.Step != "compose" || b.Part != "mesh-host" || b.Fp != "src2:new" || b.Bound != "15m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(now) || b.Owner == "" {
t.Errorf("wait-held %+v", b)
}
key := acting.Key("ace", "nats", "mesh-host")
if _, err := acting.Decode(p.w.read(buckets.Waits())[key]); err != nil {
t.Fatalf("the wait is kept: %v", err)
}
// a release by hand lets the entry move, and ends the wait once
w, _ := acting.Decode(p.w.read(buckets.Waits())[key])
w.Released = true
raw, _ := acting.Encode(w)
p.w.put(buckets.Waits(), key, raw)
if got := p.actingOf("nats", p.actingReleased())("ace"); len(got) != 0 {
t.Errorf("a released wait still holds: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted)
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != acting.HowReleased {
t.Fatalf("events %v bodies %+v", events, bodies)
}
// the version runs: the record goes, said nothing more
n.Running["ace"]["mesh-host"] = nodeengine.Running{Fingerprint: "src2:new"}
p.actingWaits(ctx, n, wanted)
if len(p.w.read(buckets.Waits())) != 0 || len(events) != 2 {
t.Errorf("kept %v events %v", p.w.read(buckets.Waits()), events)
}
}
func TestAnActingWaitOfANodeAwayStartsNoClockAndAnEntryWantedAlreadyWaitsNot(t *testing.T) {
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
p.publishWants = func(string, any, string) error { t.Fatal("said"); return nil }
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": {Node: "ace", Module: "nats", Fp: "src2:a", Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Step: "compose"}}}}
n := assign.Nodes{Online: map[string]bool{}}
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{})
n.Online["ace"] = true
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:a"}}})
if len(p.w.read(buckets.Waits())) != 0 || strings.Contains(p.graph.computed.Entries["d1"].Fp, "x") {
t.Errorf("kept %v", p.w.read(buckets.Waits()))
}
}
func TestTheToolsReadAndReleaseAnActingWait(t *testing.T) {
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
w := acting.Wait{Step: "compose", Node: "ace", Module: "nats", Part: "mesh-host", PartNode: "ace", Fp: "src2:new", Entry: "src2:a", Since: time.Now(), ClockFrom: time.Now(), Bound: "30m0s"}
raw, _ := acting.Encode(w)
s.st.put(waitsState, acting.Key("ace", "nats", "mesh-host"), raw)
got, err := s.doWaits(map[string]any{"module": "nats"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).BoundEnds == "" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now"}, "x"); err == nil {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now", "why": "the new engine is known good"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
raw, _, _, _ = s.st.get(waitsState, acting.Key("ace", "nats", "mesh-host"))
if r, _ := acting.Decode(raw); !r.Released || !strings.Contains(r.By, "known good") {
t.Errorf("wait = %+v", r)
}
}
// Design 47 G §268, A-323, A-330: the assigner and the node-engine on a node are one step: the node-engine's own want waits for no
// acting part, so it moves first, and the node's other wants, the assigner's own included, wait for the new node-engine; a want held
// on a cycle no rule breaks is the finding acting-cycle (scope module, id <node>.<module>) naming each edge's step, cleared by the
// next pass that finds it on none.
func TestTheNodeEngineIsWantedFirstAndAWantHeldOnACycleIsAFinding(t *testing.T) {
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Date(2026, 11, 13, 2, 0, 0, 0, time.UTC))
p.publishWants = func(string, any, string) error { return nil }
findings := map[string]mcseat.Finding{}
p.findings = assign.Findings{
Put: func(f mcseat.Finding) error { findings[f.Scope+"."+f.ID+"."+f.Kind] = f; return nil },
Delete: func(key string) error { delete(findings, key); return nil },
Keys: func() ([]string, error) {
var keys []string
for k := range findings {
keys = append(keys, k)
}
return keys, nil
}}
compose := meshwants.Edge{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:engine", Seat: "node-engine", Step: "compose"}
place := meshwants.Edge{Kind: "acting", Node: "ace", Module: "assigner", Fp: "src2:assigner", Seat: "mesh-wants", Step: "place"}
entries := map[string]meshwants.AssignmentEntry{
"d1": {Node: "ace", Module: "mesh-host", Fp: "src2:engine", Edges: []meshwants.Edge{place}},
"d2": {Node: "ace", Module: "assigner", Fp: "src2:assigner", Edges: []meshwants.Edge{compose}},
"d3": {Node: "ace", Module: "app", Fp: "src2:app", Edges: []meshwants.Edge{compose, place}},
}
p.graph = &graphKeeper{}
for d, e := range entries {
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: e.Node, Module: e.Module, Digest: d})
}
p.graph.computed.Entries = entries
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {
"mesh-host": {Fingerprint: "src2:old"}, "assigner": {Fingerprint: "src2:old"}}}}
wanted := map[string]map[string]mwc.Want{}
if got := p.actingOf("mesh-host", nil)("ace"); len(got) != 0 {
t.Errorf("the node-engine's want is held by %+v, want by no acting edge", got)
}
if got := p.actingOf("assigner", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
t.Errorf("the assigner's own want is held by %+v, want by the new node-engine", got)
}
p.actingWaits(context.Background(), n, wanted)
kept := p.w.read(buckets.Waits())
if kept[acting.Key("ace", "mesh-host", "assigner")] != nil || kept[acting.Key("ace", "assigner", "mesh-host")] == nil || kept[acting.Key("ace", "app", "mesh-host")] == nil {
t.Errorf("the waits kept: %v", kept)
}
if len(findings) != 0 {
t.Errorf("the cycle the node-engine's want breaks is a finding: %v", findings)
}
// Two parts holding each other across compose and place, neither the node-engine: no rule breaks it.
entries["d4"] = meshwants.AssignmentEntry{Node: "ace", Module: "x", Fp: "src2:x", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "y", Fp: "src2:y", Seat: "s-y", Step: "place"}}}
entries["d5"] = meshwants.AssignmentEntry{Node: "ace", Module: "y", Fp: "src2:y", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "x", Fp: "src2:x", Seat: "s-x", Step: "compose"}}}
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: "ace", Module: "x", Digest: "d4"},
meshwants.NodeEntryRef{Node: "ace", Module: "y", Digest: "d5"})
p.actingWaits(context.Background(), n, wanted)
f, ok := findings["module.ace.x.acting-cycle"]
if len(findings) != 2 || !ok || !strings.Contains(f.Summary, "ace/x -[place]-> ace/y -[compose]-> ace/x") {
t.Fatalf("the findings of a cycle no rule breaks: %v", findings)
}
// x's new version runs: y no longer waits, so neither is held on a cycle, and both findings clear.
n.Running["ace"]["x"] = nodeengine.Running{Fingerprint: "src2:x"}
p.actingWaits(context.Background(), n, wanted)
if len(findings) != 0 {
t.Errorf("the findings after the cycle broke: %v", findings)
}
}
@@ -1,100 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askAgainTool asks one of the assigner's asks again by hand (issue 1966, ADR 0668 §5): the recovery of an ask that ended at
// the operator channel without an answer when the event that would have asked it again was not heard.
const askAgainTool = "assigner_ask_again"
// asksState and answersState are the assigner's asks and the teller's answers, as the runtime names them.
const (
asksState = "asks"
answersState = buckets.TellerModule + ".answers"
)
// askAgainByHand writes the ask at key again under a new asking, by the rules the teller's ask-ended writes it by (assign.AskedAgain:
// a proposal's authorising ask and a put's record ask are not), so the teller asks it again; an asking the operator answered
// is refused, naming the answer. The act is recorded through the seat mesh-hand-acts.
func askAgainByHand(st store, ask func(string, any) (json.RawMessage, error), key, why, by string, now time.Time) (any, error) {
switch {
case key == "":
return nil, errors.New("name the ask's key, as the tool asks lists it")
case why == "":
return nil, errors.New("a hand ask again says why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("a hand ask again names who asks it, and the runtime proved no caller")
}
current, _, found, err := st.get(asksState, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the assigner holds no ask under %s: the tool asks lists them", key)
}
_, routerAsk, err := assign.DecodeAsk(key, current)
if err != nil {
return nil, err
}
if raw, _, found, err := st.get(answersState, key); err == nil && found {
if a, err := meshdeliveryclient.DecodeAnswer(raw); err == nil && strings.HasPrefix(a.Warrant, routerAsk.ID+"@") {
return nil, fmt.Errorf("the asking %s was answered %q (%s): it is not asked again", routerAsk.ID, a.Option, a.Words)
}
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRecord {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
if raw, _, found, err := st.get(putsState, pkey); err == nil && found {
if p, err := puts.Decode(raw); err == nil {
put = &p
}
}
}
}
value, refused := assign.AskedAgain(tellerevents.Ended{Key: key, Ask: routerAsk.ID, Outcome: occ.OutcomeCancelled}, current, put, now)
if refused != "" {
return nil, errors.New(refused)
}
if err := st.put(asksState, key, value); err != nil {
return nil, fmt.Errorf("the ask %s is not written again: %w", key, err)
}
_, next, _ := assign.DecodeAsk(key, value)
taken, rerr := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askAgainTool + " " + key, Why: why + " (" + by + ")", Cause: "ask-again"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
}
return map[string]any{"key": key, "ended": routerAsk.ID, "asked": next.ID, "by": by, "hand-act": recorded}, nil
}
// askAgainTools is the tool over the runtime's states.
func askAgainTools(st store, ask func(string, any) (json.RawMessage, error), now func() time.Time) []stdio.Tool {
return []stdio.Tool{{Name: askAgainTool, Description: "Ask one of the assigner's asks of the operator again (issue 1966): the ask " +
"under key is written again under a new asking, which the teller asks on the operator channel, as when the channel says it " +
"ended without an answer. Refused for an asking the operator answered, a proposal's authorising ask (the seat's verb again " +
"asks it) and a record ask a put asked. Recorded as a hand act.",
Input: schema(map[string]string{"key": "the ask's key, as the tool asks lists it", "why": "why it is asked again"},
[]string{"key", "why"}),
Run: func(a map[string]any) (any, error) {
key, _ := a["key"].(string)
why, _ := a["why"].(string)
return askAgainByHand(st, ask, key, why, stdio.CallerOf(a), now())
}}}
}
@@ -1,239 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/process"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askEndedSubject is the teller's event ask-ended, as the manifest's consumes names it (`mesh-delivery-teller.ask-ended`).
var askEndedSubject = process.Account{Module: buckets.TellerModule}.EventSubject(tellerevents.AskEnded)
// staleOf says why the ask at a key is stale, as the pass reads it (passer.staleAsk), or "" when it is not or when the
// trunk is not read; nil reads nothing.
type staleOf func(key string) string
// staleOfWorld is staleOf over the world the pass reads: the same rule as the pass, over the same cache.
func staleOfWorld(w *world) staleOf {
p := &passer{w: w}
return func(key string) string {
trunk := w.trunk()
if len(trunk) == 0 {
return ""
}
return p.staleAsk(key, trunk)
}
}
// hearAskEnded hears the teller's ask-ended on the assigner's one hub of its durable consumer on EVENTS (registered before
// the hub starts): an ask of the assigner's that ended at the operator channel without an answer while its key stands is
// written again under a new asking, so the teller asks it again (issue 1966, note 151). asksKV is the assigner's asks; the
// puts are opened at each event, since a put's record ask is not written again (ADR 0546).
func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, logf func(string, ...any)) error {
return hub.Handle(askEndedSubject, func(m jetstream.Msg) {
var putsKV jetstream.KeyValue
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
if err := askAgain(ctx, asksKV, putsKV, stale, m.Data(), time.Now(), logf); err != nil {
logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
})
}
// channelAsksVerb is the operator-channel seat's verb asks, read at the assigner's consumer's bind (a seat verb: no module
// named).
const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks
// readEndedAtBind registers on the hub, before it starts, one read of the operator-channel seat's asks at each bind of the
// assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155,
// ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it
// (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start.
func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, logf func(string, ...any)) {
if asksKV == nil || bus == nil {
return
}
hub.OnBound(func(created time.Time) {
go func() {
if err := endedAtBind(ctx, js, asksKV, stale, bus, wait, created, time.Now(), logf); err != nil {
logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err)
}
}()
})
}
// callWait is how a kept call the assigner waits on is said (novox/hq ADR 0668 §3): heard says the callee's machine is
// heard, now is the clock, and findings the assigner's findings, where the call unanswered past
// meshconditionsclient.CallUnansweredAfter is the finding call-unanswered while heard, cleared when it is answered.
type callWait struct {
heard func() bool
now func() time.Time
findings assign.Findings
}
// asksCallID is the finding's id for the bind's read of the operator channel's asks.
const asksCallID = "operator-channel-asks"
// keep makes call again after every transport fault (busretry: a request that reached no one, or one whose deadline
// passed) until it is answered or ctx ends, each try itself a kept call; past the bound with the callee heard it writes
// the finding call-unanswered, and clears it when answered. No timer: each try is the wait.
func (cw callWait) keep(ctx context.Context, id, callee string, call func() (json.RawMessage, error), logf func(string, ...any)) (json.RawMessage, error) {
clock := cw.now
if clock == nil {
clock = time.Now
}
since, said := clock(), false
key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered)
for {
answer, err := call()
if err == nil || !(busretry.Undelivered(err) || busretry.TimedOut(err)) || ctx.Err() != nil {
if said && cw.findings.Delete != nil {
if derr := cw.findings.Delete(key); derr != nil {
logf("the finding %s is not cleared: %v", key, derr)
}
}
return answer, err
}
if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil {
continue
}
eerr := cw.findings.Put(assign.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since).Finding(clock()))
if eerr != nil {
logf("the finding %s is not written: %v", key, eerr)
continue
}
said = true
logf("%s is unanswered since %s and its callee's machine is heard: the finding %s", callee, since.UTC().Format(time.RFC3339), key)
}
}
// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and
// each ended asking written again.
func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
keys, err := asksKV.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil
}
if err != nil {
return err
}
held := map[string][]byte{}
for _, key := range keys {
if e, err := asksKV.Get(ctx, key); err == nil {
held[key] = e.Value()
}
}
query, ok := assign.ChannelAsksQuery(held)
if !ok {
return nil
}
answer, err := wait.keep(ctx, asksCallID, channelAsksVerb, func() (json.RawMessage, error) { return bus.CallKept(channelAsksVerb, query) }, logf)
if err != nil {
return err
}
ended, cut, err := assign.EndedAtChannel(held, answer)
if err != nil {
return err
}
if cut != "" {
logf("%s at the bind: %s", channelAsksVerb, cut)
}
var putsKV jetstream.KeyValue
if js != nil {
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
}
for _, e := range ended {
body, err := tellerevents.EncodeEnded(e)
if err == nil {
err = askAgain(ctx, asksKV, putsKV, stale, body, now, logf)
}
if err != nil {
return err
}
}
logf("%s at the bind of %s: %d of %d held asks ended at the channel without an answer", channelAsksVerb, created.Format(time.RFC3339), len(ended), len(held))
return nil
}
// askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the
// asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that
// could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped.
func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, stale staleOf, data []byte, now time.Time, logf func(string, ...any)) error {
e, err := tellerevents.DecodeEnded(data)
if err != nil {
logf("the teller's %s holds what is not an ended ask, and is dropped: %v", tellerevents.AskEnded, err)
return nil
}
if asksKV == nil {
return errors.New("the assigner's asks are not open")
}
var current []byte
var rev uint64
entry, err := asksKV.Get(ctx, e.Key)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
current, rev = entry.Value(), entry.Revision()
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(e.Key); err == nil && k.Kind == asks.AskKindRecord && putsKV != nil {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
pe, err := putsKV.Get(ctx, pkey)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
if p, err := puts.Decode(pe.Value()); err == nil {
put = &p
}
}
}
}
value, why := assign.AskedAgain(e, current, put, now)
if why == "" && stale != nil {
// An ask the pass takes away is not written again: it is about a move or a composition that has been left behind, and the
// pass removes it (issue 1966 note 165).
if s := stale(e.Key); s != "" {
why = "it is stale and left to the pass to take away: " + s
}
}
if why != "" {
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is not written again: %s", e.Key, e.Ask, e.Outcome, why)
return nil
}
if _, err := asksKV.Update(ctx, e.Key, value, rev); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
logf("%s: the ask %s ended (%s) without an answer, and the key changed since: it is not written again", e.Key, e.Ask, e.Outcome)
return nil
}
return err
}
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is written again under a new asking, which the teller asks", e.Key, e.Ask, e.Outcome)
return nil
}
@@ -1,74 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §717: the assigner's asks are the seat mesh-wants' state asks, so every key its asks package writes is one of
// that state's key forms, read back to the same parts, and each value is the seat's Ask.
func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
const hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
const fp, digest = "src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
must := func(k string, err error) string {
t.Helper()
if err != nil {
t.Fatal(err)
}
return k
}
three := func(parse func(string) (string, string, string, bool)) func(string) []string {
return func(k string) []string {
if a, b, c, ok := parse(k); ok {
return []string{a, b, c}
}
return nil
}
}
for _, row := range []struct {
key string
parse func(string) []string
want []string
}{
{must(asks.HoldAskKey("web", fp)), func(k string) []string {
if a, b, ok := meshwants.ParseAsksByHoldModuleFingerprintKey(k); ok {
return []string{a, b}
}
return nil
}, []string{"web", hex}},
{must(asks.RestoreAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRestoreNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.BackupAskKey("ace", "web", fp)), three(meshwants.ParseAsksByBackupNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.OutsideAskKey("ace", "web", fp)), three(meshwants.ParseAsksByOutsideNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RevertAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRevertNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RecordAskKey("ace", "web", digest)), three(meshwants.ParseAsksByRecordNodeModuleDigestKey), []string{"ace", "web", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}},
{must(asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")), func(k string) []string {
if a, ok := meshwants.ParseAsksByAssignmentsProposalKey(k); ok {
return []string{a}
}
return nil
}, []string{"p-20261028t183000-1a2b3c"}},
} {
got := row.parse(row.key)
if len(got) != len(row.want) {
t.Errorf("%s is no key form of mesh-wants' asks: read as %v", row.key, got)
continue
}
for i := range got {
if got[i] != row.want[i] {
t.Errorf("%s reads as %v, want %v", row.key, got, row.want)
break
}
}
}
key := must(asks.HoldAskKey("web", fp))
raw, err := asks.EncodeAsk(key, asks.Ask{Ask: json.RawMessage(`{"id":"x"}`), Asking: 2})
if err != nil {
t.Fatal(err)
}
if v, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
}
}
@@ -1,170 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the
// builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's
// `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard.
// builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded).
type builtEnds struct {
mu sync.Mutex
ends map[string]builds.BuildEnded
}
func (b *builtEnds) key(module, fp string) string { return module + "@" + fp }
// put takes one build.ended in.
func (b *builtEnds) put(data []byte) error {
var e builds.BuildEnded
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a build.ended that is no end: %w", err)
}
if e.Module == "" || e.Fingerprint == "" {
return fmt.Errorf("a build's end naming no module or fingerprint: %s", data)
}
if e.Outcome == "" { // an end said before A-162 is a done build's
e.Outcome = builds.BuildDone
}
b.mu.Lock()
defer b.mu.Unlock()
if b.ends == nil {
b.ends = map[string]builds.BuildEnded{}
}
b.ends[b.key(e.Module, e.Fingerprint)] = e
return nil
}
// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended.
func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) {
b.mu.Lock()
defer b.mu.Unlock()
e, ok := b.ends[b.key(module, fp)]
return e, ok
}
// partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `<module>:<artifact>`), with the
// node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose
// parts the version's release says (releasedVersions).
func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) {
b.mu.Lock()
e, ok := b.ends[b.key(module, fp)]
b.mu.Unlock()
if !ok || (e.KeptOn == "" && e.Branch == "") {
return assign.KeptBuild{}, false
}
parts := map[string]string{}
for name, digest := range e.Made {
parts[module+":"+name] = digest
}
return assign.KeptBuild{Parts: parts, KeptOn: e.KeptOn}, true
}
// releasedVersions are the catalog's version-released heard, by module and source revision.
type releasedVersions struct {
mu sync.Mutex
seen map[string]assign.KeptBuild
// newest is, by module, the last version-released heard of it (delivered.go).
newest map[string]releaseHeard
}
func (r *releasedVersions) key(module string, source int64) string {
return fmt.Sprintf("%s@%d", module, source)
}
// put takes one version-released in.
func (r *releasedVersions) put(data []byte) error {
var e meshcatalog.VersionReleasedEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-released that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-released naming no module or version: %s", data)
}
parts := map[string]string{}
for _, a := range e.Artifacts {
parts[e.Module+":"+a.Artifact] = a.Digest
}
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]assign.KeptBuild{}
}
r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts}
if r.newest == nil {
r.newest = map[string]releaseHeard{}
}
h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source}
if e.At != nil {
h.At = *e.At
}
r.newest[e.Module] = h // the stream's order is the order of release
return nil
}
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
r.mu.Lock()
defer r.mu.Unlock()
k, ok := r.seen[r.key(module, source)]
return k, ok
}
// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and
// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703).
var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream)
// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was
// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before
// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner
// starts again.
func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) {
if err := ends.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf)
if err != nil {
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
return
}
nh.Heard(buildsStreamName)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearTaken registers the hub's taker of the indexer's take-in.taken, each waking the pass (orderwaits.go, §298, A-260); before
// the hub starts.
func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf func(string, ...any)) error {
return hub.Handle(takenSubject, func(m jetstream.Msg) {
if err := taken.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
})
}
@@ -1,110 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
)
// Design 47 G §379, §526, A-96: a build.ended heard names a kept build's or another branch's parts by digest (`<module>:<artifact>`) and its
// keeping node; a build of the trunk not kept names none, its parts being the release's.
func TestABuildEndedHeardNamesAKeptBuildsPartsAndItsNode(t *testing.T) {
put := func(b *builtEnds, e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := b.put(raw); err != nil {
t.Fatal(err)
}
}
made := map[string]string{"program": "sha256:1"}
var b builtEnds
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "a", Made: made})
if _, ok := b.partsOf("node-engine", "a"); ok {
t.Error("a trunk build not kept names parts")
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "b", Made: made, KeptOn: "ace"})
if k, ok := b.partsOf("node-engine", "b"); !ok || k.KeptOn != "ace" || !reflect.DeepEqual(k.Parts, map[string]string{"node-engine:program": "sha256:1"}) {
t.Errorf("kept build = %+v, %v", k, ok)
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "c", Made: made, Branch: "cand"})
if k, ok := b.partsOf("node-engine", "c"); !ok || k.KeptOn != "" || len(k.Parts) != 1 {
t.Errorf("branch build = %+v, %v", k, ok)
}
if err := b.put([]byte(`{"module":"x"}`)); err == nil {
t.Error("an end naming no fingerprint is kept")
}
}
// Design 47 G §379, A-96: a version-released heard names the version and the parts' digests of the module's source revision.
func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
var r releasedVersions
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := r.put(raw); err != nil {
t.Fatal(err)
}
want := assign.KeptBuild{Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}}
if k, ok := r.of("mail", 7); !ok || !reflect.DeepEqual(k, want) {
t.Errorf("released = %+v, %v", k, ok)
}
if _, ok := r.of("mail", 8); ok {
t.Error("another source revision names a version")
}
}
// Design 47 G §379, A-96: the trunk's entry takes its version and parts from the version-released of the kept assignment graph
// revision's source; a kept build from its end; no revision kept, or another fingerprint, names none.
func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *testing.T) {
w := newWorld()
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := w.released.put(raw); err != nil {
t.Fatal(err)
}
endRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "k", Made: map[string]string{"image": "sha256:2"}, KeptOn: "g14"})
if err := w.ends.put(endRaw); err != nil {
t.Fatal(err)
}
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
named := p.namedBuild("mail", "t")
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" || k.KeptOn != "" {
t.Errorf("trunk entry = %+v, %v", k, ok)
}
if k, ok := named("k"); !ok || k.KeptOn != "g14" || k.Version != "" {
t.Errorf("kept build = %+v, %v", k, ok)
}
if _, ok := named("other"); ok {
t.Error("another fingerprint names a release")
}
p.graph.source = 8
if _, ok := named("t"); ok {
t.Error("a source with no release names a version")
}
p.graph.ok = false
if _, ok := p.namedBuild("mail", "t")("t"); ok {
t.Error("no revision kept names a release")
}
}
// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed.
func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) {
w := newWorld()
say := func(e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := w.ends.put(raw); err != nil {
t.Fatal(err)
}
}
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a build nothing was heard of failed")
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"})
if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" {
t.Errorf("failed end: failed %v why %q", failed, why)
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone})
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a later done end leaves the build failed")
}
}
@@ -1,24 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G
// A-181: what was built is heard, never read from the builder's bucket).
func endDone(t testing.TB, w *world, module, fp, manifest string) {
t.Helper()
data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest})
if err != nil {
t.Fatal(err)
}
if err := w.ends.put(data); err != nil {
t.Fatal(err)
}
}
// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507).
const trunkBranch = "main"
@@ -1,50 +0,0 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// addedOf is the `added` of node's candidate (design 47 G A-316, §379, §421): by module, each module the candidate adds to node
// (laid over current) with no want there, the version and parts the want it would write carries, taken from the same code that
// writes a want's (namedBuild, the want's build at the fingerprint of the branch the want's code takes). A module whose build names none is not listed.
func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]mma.Assignments) map[string]mma.AddedModule {
if laid == nil {
return nil
}
placed := map[string]string{}
for _, m := range laid.Modules {
placed[m.Module] = m.Branch
}
var out map[string]mma.AddedModule
for _, module := range plc.ChangeOf(current[node], *laid).Added {
if key, err := mwc.WantKey(node, module); err != nil || p.w.has(buckets.Wants(), key) {
continue
}
// The branch the want's code takes: the one the candidate's placement names, else the module's default (groups, wants).
branch := p.w.trunkName()
if p.ends.following() {
branch = p.ends.branchOf(module, placed[module])
}
fp := p.w.trunkOf(branch)[module].Fp2
if fp == "" {
continue
}
k, ok := p.namedBuild(module, fp)(fp)
if !ok {
continue
}
if out == nil {
out = map[string]mma.AddedModule{}
}
out[module] = mma.AddedModule{Version: k.Version, Parts: k.Parts}
}
return out
}
// candidateOf is the candidate of proposal id for node: the assignments laid over it, and what it adds (addedOf).
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
}
@@ -1,59 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"testing"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G A-316: the candidate of a node carries, for each module it adds there without a want, the version and parts the
// want it would write carries; a module with a want there, or a build that names none, is not listed.
func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
w := newWorld()
for _, m := range []string{"mail", "web", "db"} {
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
}
w.first = "main"
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := w.released.put(rel); err != nil {
t.Fatal(err)
}
end, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: "fp-db", Made: map[string]string{"program": "sha256:3"}, KeptOn: "g14"})
if err := w.ends.put(end); err != nil {
t.Fatal(err)
}
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
c := p.candidateOf("p1", "ace", laid, current)
want := map[string]mma.AddedModule{
"mail": {Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}},
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
}
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
t.Errorf("candidate = %+v, added want %+v", c, want)
}
if got := p.candidateOf("p1", "ace", nil, current); got.Added != nil {
t.Errorf("a candidate taking the key away adds %+v", got.Added)
}
// A placement on a watched branch is added at that branch's fingerprint, as the want's code takes it, not the trunk's.
w.putModule("feat", "mail", meshcatalog.ModulesByModuleValue{Fp2: "fp-feat"}, graph.Node{}, nil)
feat, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-feat", Made: map[string]string{"program": "sha256:4"}, KeptOn: "g14"})
if err := w.ends.put(feat); err != nil {
t.Fatal(err)
}
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = w.trunkName
p.ends.on.Store(true)
onFeat := &mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "mail", Branch: "feat"}}}
if got, want := p.candidateOf("p2", "g14", onFeat, nil).Added, map[string]mma.AddedModule{"mail": {Parts: map[string]string{"mail:program": "sha256:4"}}}; !reflect.DeepEqual(got, want) {
t.Errorf("a placement on branch feat adds %+v, want %+v", got, want)
}
}
@@ -1,113 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"strings"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/assigngraph"
)
// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-<id>`'s catalog graph revision,
// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is
// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate.<group>.<heads>;
// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing.
// runGroupPrefix is a run on request's candidate group before its id, `run-<id>` (A-160).
const runGroupPrefix = "run-"
// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the
// candidate's group and heads and the revision its event names.
func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
after := ""
for {
a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after})
if err != nil {
return half, err
}
if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group {
return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision)
}
half.Revision = *a.Revision
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
after = a.Next
}
}
// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a
// group's) is passed over.
func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error {
if !strings.HasPrefix(ev.Group, runGroupPrefix) {
return nil
}
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return errors.New("the assigner keeps no assignment graph, so no candidate's is kept")
}
half, err := readCandidateHalf(call, ev)
if err != nil {
return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err)
}
in := p.graphInputs()
in.Halves = maps.Clone(in.Halves)
in.Halves[half.Revision.Branch] = half
c := assigngraph.Compute(in)
for _, u := range c.Uncomputed {
p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u)
}
rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock())
if err != nil {
return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err)
}
if fresh {
p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision)
}
return nil
}
// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235).
func (p *passer) dropCandidate(ctx context.Context, group string) error {
g := p.graph
if g == nil || g.objects == nil {
return nil
}
o, ok := g.objects.(assigngraph.Remover)
if !ok {
return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group)
}
n, err := assigngraph.DropCandidate(ctx, o, group)
if err == nil && n > 0 {
p.logf("the candidate %s's assignment graph is dropped: its run ended", group)
}
return err
}
// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it.
func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
raw, err := call(mcc.RevisionVerb, in)
if err != nil {
return meshcatalog.RevisionAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
return a, nil
}
@@ -4,12 +4,11 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
@@ -25,7 +24,7 @@ type publishedRow struct {
}
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
// names its node first): each value decoded and digested here, as the node-engine digests it (nodeengineclient.ComposedDigest).
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
// A value or key that does not read is named with why, never left out.
func published(values map[string][]byte) map[string][]publishedRow {
byNode := map[string]map[string][]byte{}
@@ -41,11 +40,7 @@ func published(values map[string][]byte) map[string][]publishedRow {
var rows []publishedRow
for key, raw := range keys {
row := publishedRow{Key: key}
_, module, part, isNode, err := nodeengineclient.ParseComposedKey(key)
var fp string
if err == nil && !isNode {
fp, err = catalogkey.ParseKeyPart(part)
}
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
switch {
case err != nil:
row.Unread = err.Error()
@@ -55,14 +50,14 @@ func published(values map[string][]byte) map[string][]publishedRow {
row.Module, row.FP = module, fp
}
if err == nil {
c, derr := nodeengineclient.DecodeComposition(raw)
c, derr := delivery.DecodeComposed(raw)
switch {
case derr != nil:
row.Unread = derr.Error()
case c.Refused != "":
row.Refused = c.Refused
default:
if d, err := nodeengineclient.ComposedDigest(c); err != nil {
if d, err := delivery.ComposedDigestOf(c); err != nil {
row.Unread = err.Error()
} else {
row.Digest = d
@@ -1,165 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"sync"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
// asked of the machine's node-engine by the seat's verb compose-candidate.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
// candidateKeyOf is the key the step's pipeline names its candidate by (A-226): a run on request's `run-<id>`, or for a branch
// pipeline the branch's catalog graph revision `<branch>.<revision>`, the branch escaped as a key part as the pipeline key does.
func candidateKeyOf(run meshwants.StepRun) (string, error) {
switch {
case run.Run != "":
return "run-" + run.Run, nil
case run.Branch == "" || run.Revision <= 0:
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
}
return keys.EscapeName(run.Branch) + "." + strconv.FormatInt(run.Revision, 10), nil
}
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
// naming none is the trunk's, trunk the first watched branch); a branch that is not watched takes the nodes placing the module on the trunk. In name order.
func machinesFor(placed map[string]map[string]string, module, branch, trunk string, watched bool) []string {
if branch == "" || !watched {
branch = trunk
}
var out []string
for node, b := range placed[module] {
if b == "" {
b = trunk
}
if b == branch {
out = append(out, node)
}
}
slices.Sort(out)
return out
}
// machinesOf are the machines of a composition step in the world's assignments.
func (w *world) machinesOf(module, branch string) []string {
return machinesFor(w.placedBranches(), module, branch, w.trunkName(), w.holdsBranch(branch))
}
// nodeInput is one machine's input: its bytes, or why the node gave none, or that it is away.
type nodeInput struct {
Node string
Body []byte
// Why is set when the node gave no input: its words, or the call's error.
Why string
// Away is a machine the watchman said away: it is not asked (§64).
Away bool
}
// compositionInputs asks the machines of a step for their inputs.
type compositionInputs struct {
machines func(module, branch string) []string
// call asks one node's seat verb (process.Conn.CallKeptOn).
call func(node, key string, body any) (json.RawMessage, error)
bound time.Duration
}
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
}
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() {
defer wg.Done()
out[i] = c.one(ctx, bound, node, key)
}()
}
wg.Wait()
return out, nil
}
// one asks one node for its input.
func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput {
in := nodeInput{Node: node}
at, cancel := context.WithTimeout(ctx, bound)
defer cancel()
type reply struct {
raw json.RawMessage
err error
}
got := make(chan reply, 1)
go func() {
raw, err := c.call(node, composeCandidateKey, ne.ComposeCandidateInput{Candidate: key, Input: true})
got <- reply{raw, err}
}()
var raw json.RawMessage
select {
case <-at.Done():
in.Why = "no answer within its bound: " + at.Err().Error()
return in
case r := <-got:
if r.err != nil {
in.Why = r.err.Error()
return in
}
raw = r.raw
}
var a ne.ComposeCandidateAnswer
// its numbers as written (json.Number), so the input kept is the engine's own bytes' values
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
if err := dec.Decode(&a); err != nil {
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
return in
}
switch {
case a.Verdict == nec.Unready:
in.Why = "unready: " + a.Refused
return in
case len(a.Input) == 0:
in.Why = "the engine answered no composition input"
if a.Refused != "" {
in.Why += ": " + a.Refused
}
return in
}
body, err := json.Marshal(a.Input)
if err != nil {
in.Why = "its composition input cannot be encoded again: " + err.Error()
return in
}
in.Body = body
return in
}
@@ -1,96 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G A-226: the candidate a step asks is `run-<id>` for a run on request, else `<branch>.<revision>`.
func TestTheCandidateKeyOfAStep(t *testing.T) {
for _, c := range []struct {
run meshwants.StepRun
want string
fail bool
}{
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
{run: meshwants.StepRun{Branch: "main"}, fail: true},
} {
got, err := candidateKeyOf(c.run)
if (err != nil) != c.fail || got != c.want {
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
}
}
}
// A-252: the machines of a step are the nodes whose placement names its branch; none named is the trunk; an unwatched branch takes
// the trunk's nodes.
func TestTheMachinesOfAStep(t *testing.T) {
placed := map[string]map[string]string{"m": {"ace": "", "g14": "feat", "novox": "main", "shanks": "feat"}, "other": {"ace": ""}}
for _, c := range []struct {
branch string
watched bool
want string
}{
{"main", true, "ace novox"},
{"feat", true, "g14 shanks"},
{"gone", false, "ace novox"},
{"", true, "ace novox"},
} {
if got := strings.Join(machinesFor(placed, "m", c.branch, trunkBranch, c.watched), " "); got != c.want {
t.Errorf("branch %q watched %v: %q, want %q", c.branch, c.watched, got, c.want)
}
}
if got := machinesFor(placed, "none", "main", trunkBranch, true); len(got) != 0 {
t.Errorf("a module placed nowhere has machines %v", got)
}
}
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// gather asks the machines side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(node, key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
mu.Lock()
asked[node] = key + " " + string(raw)
mu.Unlock()
switch node {
case "g14":
return nil, errors.New("no responders")
case "novox":
return json.RawMessage(`{"candidate":"run-r1","node":"novox","verdict":"unready","refused":"starting","modules":[],"own":{},"took":1,"v":1}`), nil
}
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, call: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
if want := `seat:node-engine.compose-candidate {"candidate":"run-r1","input":true}`; asked["ace"] != want {
t.Errorf("asked %q, want %q", asked["ace"], want)
}
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
t.Errorf("ace: %+v", got[0])
}
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
}
if _, was := asked["shanks"]; was || !got[3].Away {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, call: call}
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
@@ -1,506 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
// (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
// asks are put on and the first of their step keys (§467, §481, §600, A-163).
runAsker = meshwants.Seat
// changeCase is the one case each request composes.
changeCase = "change"
// inputVerbKey is the verb a run ask's input is served on: the seat mesh-wants' input (§354).
inputVerbKey = "seat:" + meshwants.Seat + "." + meshwants.VerbInput
// entryVerbKey and seatDefinitionsKey are the mesh-catalog seat verbs the step reads.
entryVerbKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
type runEnds struct {
mu sync.Mutex
waits map[string]chan buildstep.StepEnded
}
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
r.mu.Lock()
defer r.mu.Unlock()
if r.waits == nil {
r.waits = map[string]chan buildstep.StepEnded{}
}
ch := make(chan buildstep.StepEnded, 1)
r.waits[id] = ch
return ch
}
// forget stops waiting for the ask id.
func (r *runEnds) forget(id string) {
r.mu.Lock()
delete(r.waits, id)
r.mu.Unlock()
}
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
func (r *runEnds) heard(e buildstep.StepEnded) {
r.mu.Lock()
ch := r.waits[e.Ask]
delete(r.waits, e.Ask)
r.mu.Unlock()
if ch != nil {
ch <- e
}
}
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
type awayNodes struct {
mu sync.Mutex
away map[string]bool
}
// set keeps machine away (true) or back (false).
func (a *awayNodes) set(machine string, away bool) {
a.mu.Lock()
defer a.mu.Unlock()
if a.away == nil {
a.away = map[string]bool{}
}
if away {
a.away[machine] = true
} else {
delete(a.away, machine)
}
}
// is says machine is away.
func (a *awayNodes) is(machine string) bool {
if a == nil {
return false
}
a.mu.Lock()
defer a.mu.Unlock()
return a.away[machine]
}
// compositionRun runs a composition step.
type compositionRun struct {
inputs *compositionInputs
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
call func(key string, body any) (json.RawMessage, error)
// trunk is the trunk's catalog half as the assigner holds it.
trunk func() (mcc.Half, bool)
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
running func(node, module string) (ne.Running, bool)
away *awayNodes
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
logf func(string, ...any)
}
// machineEnd is what one machine came to.
type machineEnd struct {
failed, couldNot, warning string
}
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed, else passed with a warning per machine away or not judged.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(half.Revision.Definitions)
if err != nil {
return couldNotRun(err.Error())
}
manifests, verdicts, err := caseOf(half)
if err != nil {
return couldNotRun(err.Error())
}
programOf, err := r.programs(half)
if err != nil {
return couldNotRun(err.Error())
}
inputs, err := r.inputs.gather(ctx, run, r.away.is)
if err != nil {
return couldNotRun(err.Error())
}
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
ends := make([]machineEnd, len(inputs))
var wg sync.WaitGroup
for i, in := range inputs {
wg.Add(1)
go func() {
defer wg.Done()
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
}()
}
wg.Wait()
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
var out stepOutcome
var failed, couldNot []string
for _, e := range ends {
failed = appendSome(failed, e.failed)
couldNot = appendSome(couldNot, e.couldNot)
out.Warnings = appendSome(out.Warnings, e.warning)
}
switch {
case len(failed) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
default:
out.Outcome = meshwants.StepEndedOutcomePassed
}
return out
}
func appendSome(to []string, s string) []string {
if s == "" {
return to
}
return append(to, s)
}
func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
node := in.Node
switch {
case in.Away:
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
case in.Why != "":
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
}
program, err := programOf(node)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
raw, err := json.Marshal(req)
if err != nil {
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
}
if _, err := r.keeper.keep(ask, raw); err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
ended := r.ends.expect(ask)
defer r.ends.forget(ask)
if err := r.put(ctx, a); err != nil {
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
}
var e buildstep.StepEnded
select {
case <-ctx.Done():
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
case e = <-ended:
}
if e.Outcome != buildstep.OutcomePassed {
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
}
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
return verdictOf(node, answer)
}
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged passes with a warning.
func verdictOf(node string, answer []byte) machineEnd {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
}
if v.V != composer.Version {
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
switch {
case !ok:
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
case m.NotJudged != "":
return machineEnd{warning: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
}
return machineEnd{}
}
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
raw, err := r.call(mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(body, &a); err != nil {
return mcc.Half{}, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != run.Revision {
return mcc.Half{}, fmt.Errorf("%s answered no revision %d of %q", mcc.RevisionVerb, run.Revision, run.Branch)
}
if in.After == "" {
half.Revision = *a.Revision // the first page names what the revision was read from, its definitions included
}
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
in.After = a.Next
}
}
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
// neither holds is an error naming the seat and the digest.
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
}
def, err := json.Marshal(a.Entry.Definition)
if err != nil {
return nil, err
}
out = append(out, def)
}
return out, nil
}
// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not
// accepted (a module without one is accepted).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) {
manifests := map[string]composer.Manifest{}
verdicts := map[string]json.RawMessage{}
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
manifests[name] = composer.Manifest{Raw: raw}
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
continue
}
v := meshcatalog.VerdictsByDigestValue{V: 1, Module: name, State: e.Verdict}
if e.Verdict != mcc.VerdictRefused {
v.State = mcc.VerdictManifestWaits
}
if e.Inputs != nil {
v.Problems, v.Unknown, v.Warnings = e.Inputs.Problems, e.Inputs.Unknown, e.Inputs.Warnings
}
raw, err = json.Marshal(v)
if err != nil {
return nil, nil, err
}
verdicts[name] = raw
}
return manifests, verdicts, nil
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(mapsKeys(entries)) {
for _, s := range entries[name].Serves {
if strings.HasPrefix(s, ne.Seat+".") {
return entries[name], true
}
}
}
return meshcatalog.CatalogEntry{}, false
}
func mapsKeys(m map[string]meshcatalog.CatalogEntry) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
// programOfEntry is the composer built from a node-engine entry: its repository, commit and directory, the composer's package.
func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
dir := e.Path
if dir == "" {
dir = "."
}
if e.Repository == "" || e.Commit == "" {
return buildstep.RunProgram{}, fmt.Errorf("the node-engine's entry %s names no repository or commit", e.Fp)
}
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`.
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
}
var mu sync.Mutex
byFp := map[string]buildstep.RunProgram{}
return func(node string) (buildstep.RunProgram, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
return buildstep.RunProgram{}, fmt.Errorf("no build of %s is heard running there", engine.Module)
}
mu.Lock()
p, ok := byFp[running.Fingerprint]
mu.Unlock()
if ok {
return p, nil
}
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
return p, err
}
mu.Lock()
byFp[running.Fingerprint] = p
mu.Unlock()
return p, nil
}, nil
}
// heardPresence keeps a machine-away (away true) or machine-back of the seat mesh-watchdogs (§389, §436): the machine it names.
func heardPresence(a *awayNodes, away bool, m jetstream.Msg, logf func(string, ...any)) {
var e struct {
Machine string `json:"machine"`
}
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Machine == "" {
logf("%s holds no machine, left: %v", m.Subject(), err)
} else {
a.set(e.Machine, away)
if away {
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", e.Machine)
} else {
logf("%s is back (machine-back): a composition step composes it again", e.Machine)
}
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
}
// trunkOf is the trunk's catalog half the assigner's views hold, trunk naming the first watched branch, or false while none is
// held.
func trunkOf(iv *catalogViews, trunk func() string) func() (mcc.Half, bool) {
return func() (mcc.Half, bool) {
if iv == nil || iv.h == nil {
return mcc.Half{}, false
}
h, ok := iv.h.views()[trunk()]
return h, ok
}
}
// runAskPublisher puts a run ask on the Go build seat's check queue under the assigner's name (`accept.check.assigner`, §600), its
// id the message id, so the stream takes it once.
func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildstep.StepAsk) error {
return func(ctx context.Context, a buildstep.StepAsk) error {
raw, err := buildstep.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, a.Asker)
if err != nil {
return err
}
_, err = js.Publish(ctx, subject, raw, jetstream.WithMsgID(a.ID))
return err
}
}
@@ -1,176 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
)
const (
commitE1 = "1111111111111111111111111111111111111111"
commitE2 = "2222222222222222222222222222222222222222"
commitE3 = "3333333333333333333333333333333333333333"
)
// runStand is a composition run over stand-ins of the mesh-catalog seat, the machines' engines, the build seat and its step-ended.
type runStand struct {
mu sync.Mutex
asks map[string]buildstep.StepAsk // by node
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
}
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
engine := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
Manifest: map[string]any{"module": "engine"}}
}
revisionCommit := commitE1
if revisionEngine != "src2:e1" {
revisionCommit = commitE3
}
pages := map[string]meshcatalog.RevisionAnswer{
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
switch key {
case mcc.RevisionVerb:
var in meshcatalog.RevisionInput
_ = json.Unmarshal(raw, &in)
if in.Revision != 7 || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
return json.Marshal(pages[in.After])
case seatDefinitionsKey:
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
case entryVerbKey:
var in meshcatalog.EntryInput
_ = json.Unmarshal(raw, &in)
s.mu.Lock()
s.entries = append(s.entries, in.Fingerprint)
s.mu.Unlock()
commit := map[string]string{"src2:e1": commitE1, "src2:e2": commitE2}[in.Fingerprint]
return json.Marshal(meshcatalog.EntryAnswer{Entry: engine(in.Fingerprint, commit)})
}
return nil, errors.New("no such verb " + key)
}
engines := func(node, _ string, _ any) (json.RawMessage, error) {
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
away := &awayNodes{}
away.set("shanks", true)
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
ends := &runEnds{}
s.r = &compositionRun{
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, call: engines, bound: time.Second},
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
running: func(node, module string) (ne.Running, bool) {
fp, ok := running[node]
return ne.Running{Node: node, Module: module, Fingerprint: fp}, ok && module == "engine"
},
put: func(_ context.Context, a buildstep.StepAsk) error {
body, err := keeper.body(a.Run.Input.Digest)
if err != nil {
return err
}
req, err := composer.Decode(body)
if err != nil {
return err
}
node := req.Machines[0]
s.mu.Lock()
s.asks[node], s.requests[node] = a, req
s.mu.Unlock()
m := composer.Machine{Composes: true}
if node == "g14" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
return nil
},
}
return s
}
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
// depth compose with the one machine and one case `change`, its manifests, verdicts and seats from the step's revision; the
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
// trunk's; the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
out := s.r.run(context.Background(), "ask-step", step)
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
t.Fatalf("the step came to %+v", out)
}
if len(s.asks) != 2 {
t.Fatalf("asked %d runs, want one per machine not away", len(s.asks))
}
a, req := s.asks["ace"], s.requests["ace"]
if a.Kind != buildstep.AskCheck || a.Language != buildstep.AskGo || a.Asker != runAsker || a.Run == nil || len(a.Run.Heads) != 0 ||
a.Run.Input.Verb != "seat:mesh-wants.input" || a.Run.Program != (buildstep.RunProgram{Repository: "novox/mesh-host", Commit: commitE1, Dir: ".", Package: composer.Package}) {
t.Errorf("the run ask of ace %+v %+v", a, a.Run)
}
if s.asks["g14"].Run.Program.Commit != commitE2 {
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
}
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
t.Errorf("the request of ace %+v", req)
}
c := req.Cases[0]
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
}
if strings.Join(s.entries, " ") == "" {
t.Error("no node-engine entry is read through entry")
}
// The revision's node-engine differs from the trunk's: that build composes every machine, and no entry is asked.
changed := newRunStand(t, "src2:e3", "src2:e1", "d1")
if out := changed.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed {
t.Fatalf("the step came to %+v", out)
}
if changed.asks["ace"].Run.Program.Commit != commitE3 || changed.asks["g14"].Run.Program.Commit != commitE3 || len(changed.entries) != 0 {
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
}
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "s1") || !strings.Contains(out.Said, "d1") || len(other.asks) != 0 {
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
}
// At the inputs state's cap the ask is refused, naming the store (§354).
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
if out := full.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "the state inputs is at its cap") || len(full.asks) != 0 {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
@@ -1,168 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
// composition step of one module's pipeline that the runner asks, runs it, and says its end as the seat's event
// step-ended.<node>, the ask's id as the event's id and the event's `id`, which the runner holds the end to.
// stepOutcome is what one composition step came to.
type stepOutcome struct {
Outcome string
Said string
Warnings []string
}
// compositionSteps takes the composition steps and says their ends.
type compositionSteps struct {
node string
// say says one event of the seat mesh-wants (wantsPublisher).
say publisher
// run runs one step and answers what it came to.
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
now func() time.Time
logf func(string, ...any)
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
// accepted kind step, read from the seat list when it is said, since that subject names no asker (design 47 G A-294, which
// corrects A-267). Empty while the seat list holds no definition naming exactly one.
func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
return func() string {
d, ok := definition()
if !ok {
return ""
}
var one string
for _, m := range d.Accepts {
if m.Name != meshwants.AcceptStep || m.Callers == nil {
continue
}
for _, e := range m.Callers.List {
name, isSeat := strings.CutPrefix(e, seatdef.CallerSeatPrefix)
if !isSeat || (one != "" && one != name) {
return ""
}
one = name
}
}
return one
}
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
type stepPhase struct {
mu sync.Mutex
phase string
}
type phaseKey struct{}
// setPhase sets the phase of the step ctx runs, when it runs one.
func setPhase(ctx context.Context, phase string) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.phase = phase
p.mu.Unlock()
}
}
func (p *stepPhase) get() string {
p.mu.Lock()
defer p.mu.Unlock()
return p.phase
}
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition step, or an ask with no
// id, is left with a log line; one taken is run in its own goroutine and its end said.
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
var run meshwants.StepRun
if err := json.Unmarshal(data, &run); err != nil || id == "" || run.Step != meshwants.StepRunStepComposition {
c.logf("a step ask that is no composition step with an id is left: %v", err)
return false
}
go c.runOne(ctx, id, run)
return true
}
// runOne runs one step and says its end.
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
started := c.now().UTC()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
ended := c.now().UTC()
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
}
}
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(progressEvery)
done := make(chan struct{})
ended := make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
}
}
}()
return func() { close(done); <-ended }
}
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
func stepAskID(m *nats.Msg) string {
if id := m.Header.Get("x-event-id"); id != "" {
return id
}
return m.Header.Get(nats.MsgIdHdr)
}
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
@@ -1,224 +0,0 @@
//go:build integration
package main
import (
"bytes"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §354, §22, §63, §64, §389, A-240, A-252, A-254: the assigner started whole takes the runner's composition step of a
// module placed on three machines, one of them away: it asks the two others' node-engines for their inputs, puts one run ask per
// machine on the Go build seat's check queue under its name, its program the node-engine build each machine runs; the stand-in
// build seat reads each request by the seat mesh-wants' verb input in chunks and says its step-ended with the composer's verdict;
// the step's step-ended fails naming the machine that does not compose, warns of the machine away, and no input stays kept.
func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) {
b := startBed(t, "")
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const engineCommit = "1111111111111111111111111111111111111111"
// The assigner's durable on EVENTS hears, beside the bed's, the build seat's step-ended and the watchman's machine-away and
// machine-back, as the grants part derives them from its consumes; what each node runs it hears on its kept stream (the bed's).
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, meshwatchdogs.SubjectMachineAwayAll, meshwatchdogs.SubjectMachineBackAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
away, _ := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: "n3", At: time.Now().UTC(), LastHeard: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, meshwatchdogs.SubjectMachineAway("n3"), away); err != nil {
t.Fatal(err)
}
// The trunk's catalog half holds the module and the node-engine, the module claiming the seat node-engine.
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: engineCommit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine,
"m": {Module: "m", Fp: "src2:m1", Repository: "novox/m", Commit: engineCommit, Path: ".", Manifest: map[string]any{"module": "m", "version": "0.1"}}}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2", "n3"} {
b.place(t, node, "m")
}
// The node-engines of n1 and n2 answer compose-candidate with their inputs; n3 is never asked.
asked := make(chan string, 8)
engines, err := nc.Subscribe("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) {
node := strings.Split(m.Subject, ".")[5]
asked <- node
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"candidate": "main.3", "node": node, "verdict": "composed", "input": map[string]any{"node": node},
"modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}, "node": node})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engines.Unsubscribe() })
// mesh-catalog's entry answers the node-engine build's entry by its fingerprint.
entries, err := nc.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: engine}, "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = entries.Unsubscribe() })
subscribed(t, nc)
// The runner hears the step's end.
ended := make(chan meshwants.StepEnded, 1)
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data, &e)
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
select {
case <-b.seatListRead: // the seat server admits a caller of input once the seat list is read (A-278)
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", "step-ask-1")
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
// its end with the composer's verdict: n1 composes, n2 does not.
queue, _ := buildstep.AskStream(buildstep.AskGo)
// A-163: the assigner asks under its seat's own name.
subject := "mesh.seat.node-build-go.accept.check.mesh-wants"
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
programs := map[string]string{}
for range 2 {
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
if err != nil {
t.Fatalf("no run ask is put: %v", err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
node := req.Machines[0]
programs[node] = a.Run.Program.Commit
m := composer.Machine{Composes: true}
if node == "n2" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(v), At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
t.Fatal(err)
}
}
var e meshwants.StepEnded
select {
case e = <-ended:
case <-b.ctx.Done():
t.Fatal("the composition step said no end")
}
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
t.Fatalf("the step ended %+v", e)
}
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
t.Errorf("the composers were built at %v, want the node-engine build each machine runs", programs)
}
close(asked)
for node := range asked {
if node == "n3" {
t.Error("the machine away was asked for its input")
}
}
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
if len(keys) != 0 {
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
}
}
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
// chunks on the call's subjects, each acknowledged.
func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
t.Helper()
call := "standin" + strings.Repeat("0", 16)
chunks := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: call}
sub, err := nc.SubscribeSync(chunks.Chunks())
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
body, _ := json.Marshal(meshwants.InputInput{Digest: in.Digest, Call: call})
// On the call's subject naming its caller, the stand-in build seat's account: the seat server admits no unproven caller (A-179).
reply, err := nc.Request("mesh.seat.mesh-wants.call.input.n1~mesh-build-go", body, 10*time.Second)
if err != nil {
t.Fatal(err)
}
raw, said, ok := buscalls.Unwrap(reply.Data)
var a meshwants.InputAnswer
if !ok || json.Unmarshal(raw, &a) != nil || a.Chunks < 1 {
t.Fatalf("the verb input answered %s: %s", reply.Data, said)
}
var out bytes.Buffer
next := func() (string, []byte, error) {
m, err := sub.NextMsg(buscalls.SeatAckWithin)
if err != nil {
return "", nil, err
}
return m.Subject, m.Data, nil
}
ack := func(subject string) error { _, err := nc.Request(subject, nil, 10*time.Second); return err }
if _, err := chunks.Receive(a.Chunks, next, ack, &out); err != nil {
t.Fatal(err)
}
if err := in.Verify(out.Bytes()); err != nil {
t.Fatal(err)
}
return out.Bytes()
}
@@ -1,137 +0,0 @@
package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// saidEvents keeps what a publisher says.
type saidEvents struct {
mu sync.Mutex
said map[string]any
ids map[string]string
heard chan string
}
func newSaidEvents() *saidEvents {
return &saidEvents{said: map[string]any{}, ids: map[string]string{}, heard: make(chan string, 8)}
}
func (s *saidEvents) say(event string, body any, id string) error {
s.mu.Lock()
s.said[event], s.ids[event] = body, id
s.mu.Unlock()
s.heard <- event
return nil
}
// AS5 (§22, §354): the assigner takes mesh-wants' accepted kind `step` for the composition step and says the seat's
// step-ended.<node> with the ask's id, what the step was asked at and what it came to; an ask of another step is left.
func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
said := newSaidEvents()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
var asked meshwants.StepRun
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
t.Fatal("a step ask of another step is taken")
}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
if c.take(context.Background(), "", body) {
t.Fatal("a step ask with no id is taken")
}
if !c.take(context.Background(), "ask-1", body) {
t.Fatal("the composition step is not taken")
}
select {
case <-said.heard:
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
event := meshwants.EventStepEnded + ".n1"
end, ok := said.said[event].(meshwants.StepEnded)
if !ok || said.ids[event] != "ask-1" {
t.Fatalf("said %v under %v, want step-ended.n1 under the ask's id", said.said, said.ids)
}
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
t.Fatalf("step-ended %+v", end)
}
}
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
// asker, the phase the run is in and the time, and says none once it has ended.
func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said := newSaidEvents()
tick := make(chan time.Time)
stopped := make(chan struct{})
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
release := make(chan struct{})
// A-294: the asker is the one caller mesh-wants' definition lists for step, read from the seat list; none while unread
// or while it lists more than one.
runner := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}}
two := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner", "seat:mesh-catalog"}}}}}
for _, d := range []struct {
def seatdef.Definition
read bool
want string
}{{runner, true, "mesh-pipeline-runner"}, {runner, false, ""}, {two, true, ""}} {
if got := stepAskerOf(func() (seatdef.Definition, bool) { return d.def, d.read })(); got != d.want {
t.Errorf("the asker of %+v (read %v) = %q, want %q", d.def.Accepts, d.read, got, d.want)
}
}
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
asker: stepAskerOf(func() (seatdef.Definition, bool) { return runner, true }),
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d != time.Minute {
t.Errorf("the progress ticks every %s, want a minute", d)
}
return tick, func() { close(stopped) }
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main"})
if !c.take(context.Background(), "ask-1", body) {
t.Fatal("the composition step is not taken")
}
progress := meshwants.EventStepProgress + ".n1"
for range 2 {
tick <- at
if e := <-said.heard; e != progress {
t.Fatalf("said %s on a tick, want %s", e, progress)
}
}
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
t.Fatalf("step-progress %+v", p)
}
close(release)
if e := <-said.heard; e != meshwants.EventStepEnded+".n1" {
t.Fatalf("said %s at the end, want step-ended", e)
}
select {
case <-stopped:
default:
t.Fatal("the ticker is not stopped when the step has ended")
}
}
@@ -1,451 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// A delivery is waited for mesh-wide (design 47 G §376, §389, A-189, A-246). At its start the assigner reads whole, from their first
// message, the streams of the catalog seat's version-released and of its own version-delivered (mesh-sdk process.FromFirst, A-233),
// and the watchman's machine-away and machine-back, never its state; then it hears them. A released version of a module is said
// version-delivered on the seat mesh-wants once, when every present machine placing it on the branch it is released from runs, healthy,
// the want the assigner wrote naming that version; a version no present machine places is not delivered. A dependent whose want
// changes fingerprint waits, by an edge of kind 2, 4 or 6, for its dependency's newest released version not yet said delivered, on
// the branch it is released from; on another branch no version is released and nothing waits. The wait is said wait-held, step
// delivery, bound 60 minutes from the version's release, kept in the state delivery-waits under the dependent's name, and said
// wait-ended when the version is delivered (ran) or released by hand (assigner_release, released).
// deliveryWaitBound is a dependent's delivery wait's bound, from its dependency's version-released.
const deliveryWaitBound = 60 * time.Minute
// Wait-ended's how of a delivery wait.
const (
deliveryWaitRan = "ran"
deliveryWaitReleased = "released"
)
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
type releaseHeard struct {
Version string
Parts map[string]string
Source int64
At time.Time
}
// newestOf is the newest version of module released, the last heard of it in the stream's order.
func (r *releasedVersions) newestOf(module string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
h, ok := r.newest[module]
return h, ok
}
// deliveredVersions are the version-delivered said, the assigner's own, by module and version: each is said once.
type deliveredVersions struct {
mu sync.Mutex
said map[string]bool
}
func (d *deliveredVersions) key(module, version string) string { return module + "@" + version }
// put takes one version-delivered in.
func (d *deliveredVersions) put(data []byte) error {
var e meshwants.VersionDeliveredEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-delivered that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-delivered naming no module or version: %s", data)
}
d.mark(e.Module, e.Version)
return nil
}
func (d *deliveredVersions) mark(module, version string) {
d.mu.Lock()
defer d.mu.Unlock()
if d.said == nil {
d.said = map[string]bool{}
}
d.said[d.key(module, version)] = true
}
// has says whether version-delivered of module's version was said.
func (d *deliveredVersions) has(module, version string) bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.said[d.key(module, version)]
}
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
func presenceOf(data []byte) (string, time.Time, error) {
var e wd.MachineAwayEvent // machine-back has the same fields
if err := json.Unmarshal(data, &e); err != nil {
return "", time.Time{}, err
}
if e.Machine == "" {
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
}
return e.Machine, e.At, nil
}
// hearDelivery reads whole, then follows, the four streams a delivery is judged from: version-released, version-delivered,
// machine-away and machine-back. A stream not heard is said as the assigner's finding, and what it carries is not known until
// the assigner starts again.
func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
for _, s := range []struct {
stream, words string
take func([]byte) error
}{
{meshcatalog.StreamVersionReleased, "no want names a released version and no delivery is judged", w.released.put},
{meshwants.StreamVersionDelivered, "a version said delivered before is said again once it is judged", w.delivered.put},
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, true, at)
}
return err
}},
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, false, at)
}
return err
}},
} {
take := s.take
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
if err := take(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf, process.FromFirst())
if err != nil {
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
continue
}
nh.Heard(s.stream)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
}
// releaseBranch is the branch module's versions are released from: the first watched branch of its registration (§371).
func (p *passer) releaseBranch(module string) string {
if p.ends.following() {
return p.ends.branchOf(module, "")
}
return p.w.trunkName()
}
// placing is every machine placing module on the branch its versions are released from, sorted.
func (p *passer) placing(module string) []string {
branch := p.releaseBranch(module)
var out []string
for node, placed := range p.w.placedBranches()[module] {
if p.ends.following() {
placed = p.ends.branchOf(module, placed)
} else {
placed = p.w.trunkName() // the trunk-only form: every placement runs the trunk's
}
if placed == branch {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// deliveryStanding is where module's version stands: the present placing machines not yet running it healthy as its want names
// it, and the placing machines away.
func (p *passer) deliveryStanding(module, version string) (notRunning, away []string, placing int) {
gone := p.w.awayNow()
n := assign.Nodes{Running: p.w.runningNow()}
wanted := p.w.wantsOf()[module]
nodes := p.placing(module)
for _, node := range nodes {
if gone[node] {
away = append(away, node)
continue
}
want, ok := wanted[node]
if !ok || want.Removed || want.Version != version || !n.RunsHealthy(node, module, want.Fp) {
notRunning = append(notRunning, node)
}
}
return notRunning, away, len(nodes)
}
// judgeDelivered says version-delivered of each module's newest released version not said yet, once every present machine placing
// it runs it healthy; judged again at each pass, which each running, machine-away, machine-back and placement change wakes.
func (p *passer) judgeDelivered() {
for _, module := range p.releasedModules() {
rel, _ := p.w.released.newestOf(module)
if p.w.delivered.has(module, rel.Version) {
continue
}
notRunning, away, placing := p.deliveryStanding(module, rel.Version)
if placing == 0 || len(away) == placing || len(notRunning) > 0 {
continue
}
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s is not said, and is judged again at the next pass: %v", module, rel.Version, err)
}
}
}
// sayDelivered says version-delivered of module's release rel, with the machines away, and keeps it said.
func (p *passer) sayDelivered(module string, rel releaseHeard, away []string) error {
at := p.clock().UTC()
body := meshwants.VersionDeliveredEvent{V: keys.Version, Module: module, Version: rel.Version, Parts: rel.Parts, Away: away, At: &at}
if err := p.wantsSay(meshwants.EventVersionDelivered, body, fmt.Sprintf("%s.%s.%s.%s", meshwants.EventVersionDelivered, module, rel.Version, at.Format(time.RFC3339Nano))); err != nil {
return err
}
p.w.delivered.mark(module, rel.Version)
return nil
}
// releasedModules is every module a version-released was heard of, sorted.
func (p *passer) releasedModules() []string {
r := &p.w.released
r.mu.Lock()
defer r.mu.Unlock()
out := make([]string, 0, len(r.newest))
for m := range r.newest {
out = append(out, m)
}
sort.Strings(out)
return out
}
// deliveryWait is what a dependent waits for: its dependency, the version and the machines not yet running it.
type deliveryWait struct {
dependency string
rel releaseHeard
notRunning []string
away []string
}
// waitsForDelivery says whether module, whose want on its nodes changes to fp on branch, waits for a dependency's delivery: by
// each edge of kind 2, 4 or 6 into it in branch's catalog half, the dependency's newest released version on branch not yet said
// delivered. A want that changes no fingerprint waits for nothing; on a branch no version is released from, nothing waits.
func (p *passer) waitsForDelivery(module, fp, branch string, nodes []string) (deliveryWait, bool) {
wanted := p.w.wantsOf()[module]
changes := false
for _, node := range nodes {
if w, ok := wanted[node]; !ok || w.Removed || w.Fp != fp {
changes = true
}
}
if !changes || fp == "" {
return deliveryWait{}, false
}
var deps []string
for _, e := range p.w.edgesOf(branch, module) {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
dep, _, _ := strings.Cut(e.From, ":")
if dep != "" && dep != module && !slices.Contains(deps, dep) {
deps = append(deps, dep)
}
}
sort.Strings(deps)
for _, dep := range deps {
if p.releaseBranch(dep) != branch {
continue
}
rel, ok := p.w.released.newestOf(dep)
if !ok || p.w.delivered.has(dep, rel.Version) {
continue
}
notRunning, away, _ := p.deliveryStanding(dep, rel.Version)
return deliveryWait{dependency: dep, rel: rel, notRunning: notRunning, away: away}, true
}
return deliveryWait{}, false
}
// words is a delivery wait in plain words, as assigner_why_not and the status view say it.
func (d deliveryWait) words() string {
if len(d.notRunning) == 0 && len(d.away) > 0 {
return fmt.Sprintf("waiting for %s to be back", strings.Join(d.away, ", "))
}
return fmt.Sprintf("waiting for delivery of %s %s on %s", d.dependency, d.rel.Version, strings.Join(d.notRunning, ", "))
}
// deliveryWaitBody is the wait-held of module's delivery wait d.
func deliveryWaitBody(module, fp string, d deliveryWait, since time.Time) meshwants.Wait {
from := d.rel.At.UTC()
if from.IsZero() {
from = since.UTC()
}
return meshwants.Wait{Step: meshwants.WaitStepDelivery, Owner: meshwants.Seat, Module: module, Version: fp, Part: d.dependency,
On: d.dependency + "@" + d.rel.Version, Machines: d.notRunning, Since: since.UTC(), ClockFrom: &from, Bound: deliveryWaitBound.String(),
Words: d.words()}
}
// deliveryWaiting is one delivery wait the state delivery-waits keeps, by its dependent's name: the wait-held said, and a release
// by hand written by assigner_release.
type deliveryWaiting struct {
meshwants.Wait
// Released is set by assigner_release: the wait ends released and the dependent's want is written.
Released bool `json:"released,omitempty"`
}
// saidDeliveryWaits is every delivery wait the state delivery-waits holds, by module; a value that does not decode is said and left.
func (p *passer) saidDeliveryWaits() map[string]deliveryWaiting {
out := map[string]deliveryWaiting{}
for key, value := range p.w.read(buckets.DeliveryWaits()) {
var w deliveryWaiting
if err := json.Unmarshal(value, &w); err != nil || w.Module == "" {
p.logf("%s %s holds no delivery wait, left: %v", buckets.DeliveryWaits(), key, err)
continue
}
out[w.Module] = w
}
return out
}
// holdForDelivery says whether module's want, changing to fp on branch, is held for a dependency's delivery this pass: it says
// wait-held once and keeps it; a wait released by hand holds nothing; a wait that no longer holds is ended ran.
func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string, nodes []string) bool {
if p.kvs[buckets.DeliveryWaits()] == nil {
return false
}
key := keys.EscapeName(module)
said, was := p.saidDeliveryWaits()[module]
d, waits := p.waitsForDelivery(module, fp, branch, nodes)
switch {
case was && said.Released:
p.endDeliveryWait(ctx, key, said, deliveryWaitReleased) // the want is written now, so it changes nothing at the next pass
return false
case !waits:
if was {
p.endDeliveryWait(ctx, key, said, deliveryWaitRan)
}
return false
}
body := deliveryWaitBody(module, fp, d, p.clock())
if was && said.On == body.On && said.Version == fp && slices.Equal(said.Machines, body.Machines) {
return true
}
if was {
body.Since = said.Since
}
id := fmt.Sprintf("%s.delivery.%s.%s.%s", key, fp, body.On, strings.Join(body.Machines, ","))
if err := p.wantsSay(meshwants.EventWaitHeld, body, id); err != nil {
p.logf("the delivery wait of %s is not said, and is said at the next pass: %v", module, err)
return true
}
kept := deliveryWaiting{Wait: body}
p.put(ctx, buckets.DeliveryWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
return true
}
// endDeliveryWait says wait-ended of a delivery wait with how and removes it from the state.
func (p *passer) endDeliveryWait(ctx context.Context, key string, w deliveryWaiting, how string) {
body := w.Wait
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+".delivery."+body.Version+"."+how); err != nil {
p.logf("the end of the delivery wait of %s is not said, and is said at the next pass: %v", w.Module, err)
return
}
p.remove(ctx, buckets.DeliveryWaits(), key)
}
// againAsk is one ask of assigner_delivered_again, kept in the state delivered-again under its module (A-250): the version to say
// delivered again, who asked and why, and, while it does not hold, why not.
type againAsk struct {
Module string `json:"module"`
Version string `json:"version"`
By string `json:"by"`
Why string `json:"why"`
Waits string `json:"waits,omitempty"`
}
// releaseOf is the release heard of module's version.
func (r *releasedVersions) releaseOf(module, version string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
if h, ok := r.newest[module]; ok && h.Version == version {
return h, true
}
for key, k := range r.seen {
if strings.HasPrefix(key, module+"@") && k.Version == version {
return releaseHeard{Version: version, Parts: k.Parts}, true
}
}
return releaseHeard{}, false
}
// deliveredAgain takes each ask of assigner_delivered_again, heard at once through the watch of the state delivered-again: it says
// version-delivered again when every present machine placing the module runs that version healthy and removes the ask, else keeps
// it with why it does not hold, which assigner_waits reads.
func (p *passer) deliveredAgain(ctx context.Context) {
if p.kvs[buckets.DeliveredAgain()] == nil {
return
}
for key, value := range p.w.read(buckets.DeliveredAgain()) {
var a againAsk
if err := json.Unmarshal(value, &a); err != nil || a.Version == "" {
p.logf("%s %s holds no ask, left: %v", buckets.DeliveredAgain(), key, err)
continue
}
module := a.Module
if module == "" {
module = key
}
waits := ""
rel, ok := p.w.released.releaseOf(module, a.Version)
notRunning, away, placing := p.deliveryStanding(module, a.Version)
switch {
case !ok:
waits = fmt.Sprintf("no version-released of %s %s is heard", module, a.Version)
case placing == 0:
waits = fmt.Sprintf("no machine places %s", module)
case len(away) == placing:
waits = fmt.Sprintf("every machine placing %s is away: %s", module, strings.Join(away, ", "))
case len(notRunning) > 0:
waits = fmt.Sprintf("%s %s does not run healthy on %s", module, a.Version, strings.Join(notRunning, ", "))
}
if waits == "" {
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s asked again is not said, and is asked at the next pass: %v", module, a.Version, err)
continue
}
p.logf("version-delivered of %s %s said again, asked by %s: %s", module, a.Version, a.By, a.Why)
p.remove(ctx, buckets.DeliveredAgain(), key)
continue
}
if a.Waits != waits {
a.Waits = waits
p.put(ctx, buckets.DeliveredAgain(), key, func() ([]byte, error) { return json.Marshal(a) })
}
}
}
@@ -1,188 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §376, §389, §498, A-189, A-246: a released version is said version-delivered once, when every present machine placing
// it runs healthy the want naming it, a machine away named and not waited for, none when every placing machine is away; a dependent
// whose want changes fingerprint waits by an edge of kind 2, 4 or 6 for its dependency's newest released version not yet said
// delivered, said wait-held (step delivery, 60 minutes from the release) once and kept in delivery-waits, and wait-ended ran once
// the version is delivered; a kind-1 edge, or a dependency with no version released, waits for nothing; one released by hand ends
// released and its want is written.
func TestADependentWaitsForItsDependencysDelivery(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
waits := &memKV{values: map[string][]byte{}}
again := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.DeliveryWaits(): waits, buckets.DeliveredAgain(): again}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
p.w.runningHeard = true
placeOn(t, p.w, "ace", "db", "app")
placeOn(t, p.w, "g14", "db")
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
want := func(node, module, fp, version string) {
key, _ := mwc.WantKey(node, module)
raw, err := mwc.EncodeWant(mwc.Want{Fp: fp, Version: version, Self: true})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Wants(), key, raw)
}
runs := func(node, module, fp string) {
raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "fingerprint": fp, "at": at})
if err := p.w.hearRunning("running", raw); err != nil {
t.Fatal(err)
}
}
source := int64(3)
release := func(module, version string) {
source++
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{V: 1, Module: module, Version: version, Source: source, At: &at,
Artifacts: []meshcatalog.VersionArtifact{{Artifact: "program", Digest: "sha256:d"}}})
if err := p.w.released.put(raw); err != nil {
t.Fatal(err)
}
}
saidOf := func(event string) []said {
var out []said
for _, e := range events {
if e.event == event {
out = append(out, e)
}
}
return out
}
want("ace", "app", fp("1"), "")
// No version of db is released: app's new fingerprint waits for nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app waits with no version of db released")
}
release("db", "1.1.0")
want("ace", "db", fp("d"), "1.1.0")
want("g14", "db", fp("d"), "1.1.0")
runs("ace", "db", fp("d"))
// A kind-1 edge orders nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 1, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("a kind-1 edge holds app")
}
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
// app's want unchanged waits for nothing.
if p.holdForDelivery(ctx, "app", fp("1"), trunkBranch, []string{"ace"}) {
t.Fatal("app's unchanged want waits")
}
// g14 places db and does not run 1.1.0 yet: not delivered, and app's new version waits, said once.
p.judgeDelivered()
if !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) || !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.1.0")
}
if len(saidOf(meshwants.EventVersionDelivered)) != 0 {
t.Fatalf("db is said delivered while g14 does not run it: %+v", events)
}
held := saidOf(meshwants.EventWaitHeld)
if len(held) != 1 {
t.Fatalf("wait-held said %d times, want once", len(held))
}
var w meshwants.Wait
_ = json.Unmarshal(held[0].body, &w)
if w.Step != meshwants.WaitStepDelivery || w.Part != "db" || w.Module != "app" || w.Version != fp("2") || w.Bound != "1h0m0s" ||
w.ClockFrom == nil || !w.ClockFrom.Equal(at) || len(w.Machines) != 1 || w.Machines[0] != "g14" || w.Words != "waiting for delivery of db 1.1.0 on g14" {
t.Errorf("the wait said is %+v", w)
}
if _, ok := waits.values["app"]; !ok {
t.Errorf("the wait is not kept in delivery-waits: %v", waits.values)
}
// g14 away: every present machine runs it, so it is delivered, g14 named away; once only.
p.w.heardPresence("g14", true, at)
p.judgeDelivered()
p.judgeDelivered()
delivered := saidOf(meshwants.EventVersionDelivered)
var d meshwants.VersionDeliveredEvent
if len(delivered) != 1 || json.Unmarshal(delivered[0].body, &d) != nil || d.Module != "db" || d.Version != "1.1.0" ||
len(d.Away) != 1 || d.Away[0] != "g14" || d.Parts["db:program"] != "sha256:d" {
t.Fatalf("version-delivered said %+v", delivered)
}
// The wait ends ran, and app's want goes on.
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app still waits once db 1.1.0 is delivered")
}
ended := saidOf(meshwants.EventWaitEnded)
if len(ended) != 1 || json.Unmarshal(ended[0].body, &w) != nil || w.How != deliveryWaitRan {
t.Errorf("wait-ended said %+v", ended)
}
if _, ok := waits.values["app"]; ok {
t.Error("the ended wait is kept")
}
// A newer release with every placing machine away is not delivered, and its wait says the machines it waits to be back.
release("db", "1.2.0")
p.w.heardPresence("ace", true, at)
p.judgeDelivered()
if len(saidOf(meshwants.EventVersionDelivered)) != 1 {
t.Error("a version is said delivered while every placing machine is away")
}
if !p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.2.0")
}
held = saidOf(meshwants.EventWaitHeld)
if json.Unmarshal(held[len(held)-1].body, &w) != nil || w.Words != "waiting for ace, g14 to be back" {
t.Errorf("the wait while every machine is away says %q", w.Words)
}
// Released by hand (assigner_release writes it): the wait ends released and app's want is written.
var kept deliveryWaiting
_ = json.Unmarshal(waits.values["app"], &kept)
kept.Released = true
raw, _ := json.Marshal(kept)
p.w.put(buckets.DeliveryWaits(), "app", raw)
if p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("a released wait holds app")
}
ended = saidOf(meshwants.EventWaitEnded)
if json.Unmarshal(ended[len(ended)-1].body, &w) != nil || w.How != deliveryWaitReleased {
t.Errorf("the released wait ended %+v", w)
}
// Presence: the later of machine-away and machine-back holds, whichever stream is read first.
p.w.heardPresence("ace", false, at.Add(time.Minute))
p.w.heardPresence("ace", true, at)
if p.w.awayNow()["ace"] || !p.w.online()["ace"] {
t.Error("an older machine-away read after a newer machine-back leaves ace away")
}
// A-250: an ask of assigner_delivered_again of 1.1.0, which ace (present) runs and g14 (away) does not, is said again and taken
// away; one of 1.2.0, which no machine runs, is kept with why.
ask := func(version string) {
raw, _ := json.Marshal(againAsk{Module: "db", Version: version, By: "op", Why: "the catalog missed it"})
p.w.put(buckets.DeliveredAgain(), "db", raw)
}
before := len(saidOf(meshwants.EventVersionDelivered))
ask("1.1.0")
p.deliveredAgain(ctx)
if len(saidOf(meshwants.EventVersionDelivered)) != before+1 || p.w.has(buckets.DeliveredAgain(), "db") {
t.Errorf("the ask of 1.1.0 said %d version-delivered and is kept %v", len(saidOf(meshwants.EventVersionDelivered))-before, p.w.has(buckets.DeliveredAgain(), "db"))
}
ask("1.2.0")
p.deliveredAgain(ctx)
var a againAsk
if json.Unmarshal(p.w.value(buckets.DeliveredAgain(), "db"), &a) != nil || a.Waits != "db 1.2.0 does not run healthy on ace" ||
len(saidOf(meshwants.EventVersionDelivered)) != before+1 {
t.Errorf("the ask of 1.2.0 = %+v", a)
}
}
@@ -1,82 +0,0 @@
package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
)
// runtimeObjects is the kept data assignment-revisions as the bundle reaches it through its runtime (the old objects
// section of a foundation part, design 47 lines 1362-1366).
type runtimeObjects struct{ o busstate.Objects }
func (r runtimeObjects) Put(_ context.Context, name string, data []byte) error {
_, err := r.o.Put(name, data, "")
return err
}
func (r runtimeObjects) Get(_ context.Context, name string) ([]byte, error) {
d, err := r.o.Get(name)
if err != nil {
return nil, err
}
if d.Info.Name == "" || d.Info.Deleted {
return nil, assigngraph.ErrNoObject
}
return d.Data, nil
}
func (r runtimeObjects) Names(context.Context) ([]string, error) {
infos, err := r.o.List()
if err != nil {
return nil, err
}
names := []string{}
for _, i := range infos {
if !i.Deleted {
names = append(names, i.Name)
}
}
return names, nil
}
// graphVerbs are the seat mesh-wants' verbs revision, entry and difference (design 47 lines 1365-1366: an assignment graph
// revision is served by the seat's verbs), each reading only the kept revisions.
func graphVerbs(o assigngraph.Objects) map[string]func(map[string]any) (any, error) {
return map[string]func(map[string]any) (any, error){
meshwants.VerbRevision: func(a map[string]any) (any, error) {
var in meshwants.RevisionInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Revision(context.Background(), o, in)
},
meshwants.VerbEntry: func(a map[string]any) (any, error) {
var in meshwants.EntryInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Entry(context.Background(), o, in)
},
meshwants.VerbDifference: func(a map[string]any) (any, error) {
var in meshwants.DifferenceInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Difference(context.Background(), o, in)
},
}
}
// decodeArgs reads a verb's arguments into its input type; a caller's own fields beside them are the runtime's.
func decodeArgs(a map[string]any, into any) error {
raw, err := json.Marshal(a)
if err != nil {
return err
}
return json.Unmarshal(raw, into)
}
@@ -1,82 +0,0 @@
package main
import (
"context"
"fmt"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
f, is := g.objects.(assigngraph.Filler)
if !is {
return 0, 0, false
}
used, capacity, err := f.Fill(ctx)
return used, capacity, err == nil && capacity > 0
}
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
at := p.clock().UTC()
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
}
}
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
// its cap), so a fill reached while the process was down is said too.
func (p *passer) fillAtStart(ctx context.Context) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return
}
switch {
case used >= capacity:
g.full = true
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
case used*5 >= capacity*4:
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
}
}
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
// 80 % again. It answers true while the store is full.
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return false
}
if g.full && used*5 < capacity*4 {
g.full = false
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
}
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
g.full = false
return false
}
if !g.full {
g.full = true
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
}
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
return true
}
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
func (p *passer) fillingAfter(ctx context.Context, before int64) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
}
}
@@ -1,83 +0,0 @@
package main
import (
"context"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
type filled struct {
objs map[string][]byte
base int64
cap int64
}
func (f *filled) Put(_ context.Context, name string, data []byte) error {
f.objs[name] = data
return nil
}
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := f.objs[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (f *filled) Names(context.Context) ([]string, error) {
out := []string{}
for n := range f.objs {
out = append(out, n)
}
return out, nil
}
func (f *filled) Fill(context.Context) (int64, int64, error) {
used := f.base
for _, d := range f.objs {
used += int64(len(d))
}
return used, f.cap, nil
}
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
ctx := context.Background()
p := newPasser(kvSet{}, time.Now())
var events []string
p.publishWants = func(event string, body any, id string) error {
if _, ok := body.(meshwants.Fill); ok {
events = append(events, event)
}
return nil
}
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
if len(events) != 0 {
t.Fatalf("events %v", events)
}
o.base = 700
p.graph.started = false
p.keepGraph(ctx)
if len(events) != 1 || events[0] != meshwants.EventFilling {
t.Fatalf("at the start past 80 %%: %v", events)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
o.base = 990
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
t.Fatalf("a revision that does not fit: events %v", events)
}
p.fullBefore(ctx, c)
if n := len(events); n != 2 {
t.Errorf("full is said once: %v", events)
}
o.base = 100
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
t.Errorf("room: events %v", events)
}
}
@@ -1,192 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
// from the catalog halves it follows, the placements, the first branch of each registration and each node's capability reading,
// keeps it in the objects assignment-revisions and says the seat mesh-wants' event assignment-revision; the wants of the pass
// that follows are written from the revision it holds.
type graphKeeper struct {
objects assigngraph.Objects
// halves is every followed branch's catalog graph revision held now.
halves func() map[string]mcc.Half
// last is the revision last computed, so a pass whose inputs did not change keeps and says nothing.
last meshwants.AssignmentRevision
has bool
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
kept meshwants.AssignmentRevision
source int64
ok bool
said string
// cycled is the cycles last said (assigngraph.Cycles).
cycled string
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
full, started bool
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
pins *pinKeeper
// computed is the revision computed by the last pass, with its entries: the edges the wants wait on (actingwaits.go).
computed assigngraph.Computed
}
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
func (p *passer) keepGraph(ctx context.Context) {
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return
}
c := assigngraph.Compute(p.graphInputs())
g.computed = c
if p.ends != nil {
p.ends.setRunner(assigngraph.Runner(slices.Collect(maps.Values(c.Entries)))) // the runner's member of a group goes first (A-334)
}
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
p.logf("the assignment graph leaves out %s", u)
}
}
if cycles := strings.Join(c.Cycles, "; "); cycles != g.cycled {
g.cycled = cycles
if cycles != "" {
p.logf("the assignment graph holds a cycle among the acting waits, within one step or across steps: %s", cycles)
}
}
if !g.started {
g.started = true
p.fillAtStart(ctx)
}
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
return
}
used, _, _ := g.fillOf(ctx)
if p.fullBefore(ctx, c) {
return
}
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
if err != nil {
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
return
}
g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true
g.source = 0
for _, cat := range rev.Catalogs {
if cat.Branch == p.w.trunkName() {
g.source = cat.Source
}
}
if !fresh {
return
}
p.fillingAfter(ctx, used)
if g.pins != nil {
g.pins.judge() // the revision before ceased to be current
}
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
}
}
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
// node's capability reading and the first branch of each registration.
func (p *passer) graphInputs() assigngraph.Inputs {
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
// (design 47 G §708).
if r := p.seats.Reader(); r != nil {
in.Definitions = r.Definitions()
}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
first := p.w.trunkName()
in.FirstBranch = func(string) string { return first }
if p.ends != nil {
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
}
return in
}
// capabilityDigests is each node's capability reading, by node: the statement its engine made of what it reads, and the
// capabilities its tool runner states with the fingerprint of the runner that announced them, as the node's record carries
// them (design 46 approval mark §6, lead answer A-203), the digest over all of it.
func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem {
records := p.w.records()
known := p.w.knownNodes()
slices.Sort(known)
var out []meshwants.AssignmentRevisionCapabilitiesItem
for _, node := range known {
read := p.reads != nil && p.reads.Known(node)
rec, recorded := records[node]
if !read && !recorded {
continue
}
var reads []string
if read {
reads = slices.Clone(p.reads.Reads([]string{node})[node])
slices.Sort(reads)
}
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
Fingerprint: rec.RunnerFingerprint}
slices.Sort(item.Readings)
raw, _ := json.Marshal(struct {
Reads []string `json:"reads"`
Readings []string `json:"readings"`
Fingerprint string `json:"fingerprint"`
}{reads, item.Readings, item.Fingerprint})
sum := sha256.Sum256(raw)
item.Digest = "sha256:" + hex.EncodeToString(sum[:])
out = append(out, item)
}
return out
}
// sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is
// said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there.
func (p *passer) sayWanted(node, module, fp string, removed bool) {
if p.publishWants == nil {
return
}
g := p.graph
if g == nil || !g.ok {
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
return
}
at := p.clock().UTC()
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at}
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, g.kept.Revision, at.Format(time.RFC3339Nano))
if err := p.wantsSay(event, body, id); err != nil {
p.logf("wanted of %s on %s is not said: %v", module, node, err)
}
}
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
// version and the parts of the `version-released` whose source is the one the kept assignment graph revision was computed from (A-67).
func (p *passer) namedBuild(module, trunkFP string) func(fp string) (assign.KeptBuild, bool) {
return func(fp string) (assign.KeptBuild, bool) {
if k, ok := p.w.ends.partsOf(module, fp); ok {
return k, true
}
if fp != trunkFP || p.graph == nil || !p.graph.ok {
return assign.KeptBuild{}, false
}
return p.w.released.of(module, p.graph.source)
}
}
@@ -1,134 +0,0 @@
package main
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// said is one event the passer said on the seat mesh-wants.
type said struct {
event, id string
body json.RawMessage
}
// Design 47 G A-67, §292: the pass keeps an assignment graph revision per catalog revision taken in, before it writes any want, and
// says assignment-revision; A-61, §693: a want written or removed is said `wanted` on the subject of its node and module in the same
// write, from the kept revision and its catalog's source, a put that wrote nothing says nothing, and with no revision kept none is said.
func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
kv := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): kv}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
placeOn(t, p.w, "ace", "web")
// Design 46 approval mark §6: a node's reading keeps the fingerprint of the runner that announced it, as its record says.
rec, _ := json.Marshal(nodeengine.HaveByNodeNodeValue{V: 1, Record: &nodeengine.NodeRecord{V: 1, Node: "ace",
RunnerCapabilities: []string{"bus-state-calls", "approval-mark/1"}, RunnerFingerprint: "sha256:runner"}})
p.w.put(buckets.Have, "ace.node", rec)
if c := p.capabilityDigests(); len(c) != 1 || c[0].Node != "ace" || c[0].Fingerprint != "sha256:runner" ||
!slices.Equal(c[0].Readings, []string{"approval-mark/1", "bus-state-calls"}) || !strings.HasPrefix(c[0].Digest, "sha256:") {
t.Errorf("the capability readings are %+v", c)
}
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fpTest}}}
objects := memObjects{}
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} }}
want := mwc.Want{Fp: fpTest, Self: true}
p.putWant(ctx, "ace", "web", want) // no revision kept yet: written, and not said
if len(events) != 0 || kv.puts != 1 {
t.Fatalf("with no revision kept: %d events, %d puts", len(events), kv.puts)
}
key, _ := mwc.WantKey("ace", "web")
delete(p.w.read(buckets.Wants()), key) // the world forgets the want, as before the first pass
kv.puts = 0
p.w.put(buckets.Wants(), key, nil)
p.keepGraph(ctx)
if len(events) != 1 || events[0].event != meshwants.EventAssignmentRevision || len(objects) == 0 {
t.Fatalf("the pass kept %d objects and said %+v", len(objects), events)
}
p.keepGraph(ctx) // the same inputs: nothing kept, nothing said
if len(events) != 1 {
t.Fatalf("a pass of the same inputs said %d events", len(events))
}
p.putWant(ctx, "ace", "web", want)
p.putWant(ctx, "ace", "web", want) // unchanged: not written, not said
if len(events) != 2 || kv.puts != 1 {
t.Fatalf("a want put twice: %d events, %d puts", len(events), kv.puts)
}
if wantEvent := "wanted.ace.web"; events[1].event != wantEvent {
t.Errorf("wanted said on %q, want %q", events[1].event, wantEvent)
}
var w meshwants.WantedEvent
if err := json.Unmarshal(events[1].body, &w); err != nil || w.Node != "ace" || w.Module != "web" || w.Fp != fpTest || w.Removed || w.Revision != 1 || w.Source != 9 {
t.Errorf("wanted = %+v (%v)", w, err)
}
p.putWant(ctx, "ace", "web", mwc.Want{Removed: true})
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
}
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
// keeps none, and the run's group-ended drops it.
before, puts := len(events), kv.puts
ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5}
var asked []meshcatalog.RevisionInput
call := func(key string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.RevisionInput)
asked = append(asked, in)
rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}}
a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}}
return json.Marshal(a)
}
if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 {
t.Errorf("a try's candidate asked %v: %v", asked, err)
}
if err := p.keepCandidate(ctx, call, ev); err != nil {
t.Fatal(err)
}
if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 {
t.Errorf("the candidate's revision was asked as %+v", asked)
}
a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"})
if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 {
t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err)
}
if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the run's entry of web is %+v: %v", e.Entry, err)
}
if len(events) != before || kv.puts != puts {
t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts)
}
if err := p.dropCandidate(ctx, "run-7"); err != nil {
t.Fatal(err)
}
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok {
t.Error("the run's group-ended left its candidate kept")
}
if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil {
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
}
}
@@ -1,163 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"slices"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
)
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the
// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name
// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each
// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings;
// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
// half, never its own output back (§195).
type halves struct {
w *world
call mcc.Call
reader *views.ProcessReader
wake func()
logf func(string, ...any)
ctx context.Context
branches *mcc.Branches
mu sync.Mutex
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
settled chan<- struct{}
told bool
}
// follow makes the branch follower on the reader; the reader's start asks the first views.
func (h *halves) follow() error {
b, err := mcc.FollowBranchesProcess(h.reader, h.call)
if err != nil {
return err
}
b.Logf = h.logf
b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
}
b.Adopted = h.adopted
h.mu.Lock()
h.branches = b
h.mu.Unlock()
return nil
}
// first is the first watched branch followed, the trunk's: "" while none is.
func (h *halves) first() string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return ""
}
if f := b.Followed(); len(f) > 0 {
return f[0]
}
return ""
}
// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its
// first view.
func (h *halves) adopted(branch string, c *mcc.Catalog) {
first := h.first()
if first == "" {
return
}
h.w.setTrunk(first)
h.wake()
if branch != first || h.settled == nil {
return
}
h.mu.Lock()
told := h.told
h.told = true
h.mu.Unlock()
if told {
return
}
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
}()
}
// changed takes one entry of branch's half into the world, or a module taken out of it.
func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) {
defer h.wake()
if deleted {
h.w.dropModule(branch, module)
return
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(raw, &e); err != nil {
h.logf("the entry of %s on branch %q does not read: %v", module, branch, err)
return
}
v, n, m := fromEntry(e)
h.w.putModule(branch, module, v, n, m)
h.w.putEdges(branch, module, e.Edges)
}
// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of
// its words.
func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) {
v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages,
Path: e.Path, Repository: e.Repository, Verdict: e.Verdict}
n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires,
Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
for p := range e.Packages {
n.Packages = append(n.Packages, p)
}
slices.Sort(n.Packages)
if len(e.Manifest) == 0 {
return v, n, nil
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return v, n, nil
}
sum := sha256.Sum256(raw)
v.Manifest = "sha256:" + hex.EncodeToString(sum[:])
man, err := catalogue.Decode(raw)
if err != nil {
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
}
n.Serves = graph.ServesOf(man)
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
return v, n, &parsed{module: e.Module, manifest: man}
}
// views is the catalog graph revision held now of each followed branch whose first view is answered.
func (h *halves) views() map[string]mcc.Half {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
out := map[string]mcc.Half{}
if b == nil {
return out
}
for _, branch := range b.Followed() {
if c, ok := b.Catalog(branch); ok {
if half, ok := c.View(); ok {
out[branch] = half
}
}
}
return out
}
@@ -1,377 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
const inputsTool = "assigner_inputs"
const (
// inputsKeptFor is how long an input is kept at the longest.
inputsKeptFor = 24 * time.Hour
// inputsSwept is how many of the oldest keys each new ask judges.
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the input's own key, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
)
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
// inputsCap is the state inputs' cap (design 47 G §354's kept-data row: 64 MiB, the manifest's max-mib): at it, an ask is refused.
const inputsCap = 64 << 20
// inputKeeper keeps the inputs over the state and its object store.
type inputKeeper struct {
vals stateobjects.Values
now func() time.Time
logf func(string, ...any)
// used is how many bytes the state and its object store hold; nil judges no cap.
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
}
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
if k.used != nil {
used, err := k.used()
if err != nil {
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
digest := digestOf(body)
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
// input a sweep deleted, changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
}
askKey := inputsAskPrefix + keys.EscapeName(ask)
e, err := k.vals.KV.Get(askKey)
if err != nil || e == nil {
return err
}
if err := k.vals.KV.Delete(string(e.Value)); err != nil {
return err
}
return k.vals.KV.Delete(askKey)
}
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
var inputs, asks []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
asks = append(asks, key)
} else {
inputs = append(inputs, key)
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > inputsSwept {
inputs = inputs[:inputsSwept]
}
for _, key := range inputs {
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
continue
}
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
for _, key := range asks {
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
}
}
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none.
func (k *inputKeeper) keyOf(digest string) (string, error) {
all, err := k.vals.KV.Keys()
if err != nil {
return "", err
}
slices.Sort(all)
for _, key := range all {
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
return key, nil
}
}
return "", nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
func (k *inputKeeper) body(digest string) ([]byte, error) {
if len(digest) != 64 {
return nil, errors.New("input takes the input's sha256, 64 hex digits")
}
key, err := k.keyOf(digest)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
}
if key == "" {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
var text string
found, err := k.vals.Get(key, &text)
if err != nil {
return nil, fmt.Errorf("the input %s cannot be read: %w", key, err)
}
if !found {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
b, err := base64.StdEncoding.DecodeString(text)
if err != nil || digestOf(b) != digest {
return nil, fmt.Errorf("the input %s does not hold the bytes its digest names", key)
}
return b, nil
}
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
// verb answers the seat's verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
// size. A caller that does not acknowledge within `within` ends the send, said in logf.
func (k *inputKeeper) verb(line buscalls.ChunkLine, within time.Duration) process.SeatHandler {
return func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.InputInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
if !callToken.MatchString(in.Call) {
return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
}
digest := strings.TrimPrefix(strings.TrimSpace(in.Digest), "sha256:")
body, err := k.body(digest)
if err != nil {
return nil, err
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: in.Call}
total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
go func() {
if err := send.Send(line, body, within, k.logf); err != nil {
k.logf("the send %s of the input %s: %v", in.Call, digest, err)
}
}()
return meshwants.InputAnswer{Digest: digest, Chunks: total}, nil
}
}
// inputObjects is the object store beside the state inputs (`inputs-objects`, a state's own, §712), opened when first used: until
// the grants part has made it every call is process.ErrNotYet.
type inputObjects struct {
js jetstream.JetStream
module string
mu sync.Mutex
bus stateobjects.Objects
}
func (o *inputObjects) open() (stateobjects.Objects, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.bus != nil {
return o.bus, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object store of the state inputs is not open: %w", err)
}
o.bus = stateobjects.BusObjects{Store: store}
return o.bus, nil
}
func (o *inputObjects) MostBytes() int { return 0 }
func (o *inputObjects) Put(name string, data []byte) error {
b, err := o.open()
if err != nil {
return err
}
return b.Put(name, data)
}
func (o *inputObjects) Get(name string) ([]byte, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.Get(name)
}
func (o *inputObjects) Delete(name string) error {
b, err := o.open()
if err != nil {
return err
}
return b.Delete(name)
}
func (o *inputObjects) List() ([]stateobjects.Object, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.List()
}
// inputsOver is the inputs the program keeps, over its bucket and its object store beside it; nil when the bucket is not opened.
func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, logf func(string, ...any)) *inputKeeper {
if kv == nil {
return nil
}
objects := &inputObjects{js: js, module: module}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
used := func() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
n := int64(st.Bytes())
objs, err := objects.List()
if err != nil && !errors.Is(err, process.ErrNotYet) {
return 0, err
}
for _, o := range objs {
n += o.Size
}
return n, nil
}
return &inputKeeper{vals: v, now: time.Now, logf: logf, used: used}
}
// inputSaid is one kept input as assigner_inputs lists it.
type inputSaid struct {
Key string `json:"key"`
Time string `json:"time"`
Digest string `json:"digest"`
Size int `json:"size"`
Input string `json:"input,omitempty"`
}
// inputsRead is the tool assigner_inputs over the kept inputs: every kept input with its size, or the one named by digest whole.
func inputsRead(k *inputKeeper) stdio.Tool {
return stdio.Tool{Name: inputsTool, Description: "The inputs the assigner keeps for the composition steps it asked a build seat to run (design 47 G §354): each " +
"with its write time, digest and size, or one by `digest` whole (base64). Read-only.",
Input: schema(map[string]string{"digest": "one input's sha256"}, nil), Run: func(args map[string]any) (any, error) {
want := arg(args, "digest")
all, err := k.vals.KV.Keys()
if err != nil {
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
}
slices.Sort(all)
out := []inputSaid{}
for _, key := range all {
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
continue
}
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
}
said := inputSaid{Key: key, Time: t, Digest: d, Size: base64.StdEncoding.DecodedLen(len(text))}
if b, err := base64.StdEncoding.DecodeString(text); err == nil {
said.Size = len(b)
}
if want != "" {
said.Input = text
}
out = append(out, said)
}
return map[string]any{"inputs": out}, nil
}}
}
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := k.ended(e.Ask); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
if ends != nil {
ends.heard(e)
}
_ = m.Ack()
}
@@ -1,231 +0,0 @@
package main
import (
"bytes"
"crypto/rand"
"encoding/json"
"errors"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (k *kvMem) Get(key string) (*stateobjects.Entry, error) {
k.mu.Lock()
defer k.mu.Unlock()
v, ok := k.vals[key]
if !ok {
return nil, nil
}
return &stateobjects.Entry{Value: v, Revision: k.revs[key]}, nil
}
func (k *kvMem) Create(key string, v []byte) error {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, 1
return nil
}
func (k *kvMem) Update(key string, v []byte, rev uint64) error {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, rev+1
return nil
}
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
delete(k.vals, key)
return nil
}
func (k *kvMem) Keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
sort.Strings(out)
return out, nil
}
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(string) error { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
func testKeeper(objs *objectsMem, now func() time.Time) *inputKeeper {
return &inputKeeper{vals: stateobjects.Values{KV: newKVMem(), Objects: objs, Store: "inputs-objects"}, now: now, logf: func(string, ...any) {}}
}
// Design 47 G §354, §712, A-237: an input of any size is kept as an object beside the state, and the seat mesh-wants' verb input
// answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, time.Now)
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := k.keep("ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := k.body(digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := k.verb(line, time.Second)
raw, _ := json.Marshal(meshwants.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshwants.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+strings.Repeat("b", 64)+`","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// assigner_inputs lists it with its size and gives one whole by digest
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["inputs"].([]inputSaid)) != 1 || list.(map[string]any)["inputs"].([]inputSaid)[0].Size != len(body) {
t.Errorf("assigner_inputs = %+v %v", list, err)
}
}
// §354: an input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, judged at each new ask.
func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
now := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, func() time.Time { return now })
d1, err := k.keep("ask/one", []byte(`{"heads":"abc"}`))
if err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("a kept input is not read: %v", err)
}
if err := k.ended("ask/other"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("the step-ended of another ask deleted it: %v", err)
}
if err := k.ended("ask/one"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err == nil {
t.Error("the input of an ended ask is still answered")
}
if keys, _ := k.vals.KV.Keys(); len(keys) != 0 {
t.Errorf("keys left after the step-ended: %v", keys)
}
d2, _ := k.keep("ask/two", []byte(`{"heads":"def"}`))
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask/three", []byte(`{"heads":"ghi"}`)); err != nil {
t.Fatal(err)
}
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
}
File diff suppressed because it is too large Load Diff
@@ -8,42 +8,29 @@ package main
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"io"
"log"
"maps"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"os"
"os/signal"
"slices"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
func main() {
@@ -67,160 +54,109 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buckets.Grants,
buckets.Have,
buckets.Provisions,
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
provisions.ProvisionsBucket,
delivery.SourceBucket,
delivery.MergesBucket,
graph.Bucket,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
var readFamilies = []string{buckets.ComposedRead}
var readFamilies = []string{delivery.ComposedRead}
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-module-assigner's three among them
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
const engineReadsState = "engine-reads"
// engineReadsBucket is that state's bucket.
func engineReadsBucket() string { return assignment.Module() + "_" + engineReadsState }
// writtenBucket is the bucket of the state written (pipelines.go): the pipeline ends the wants follow.
func writtenBucket() string { return assignment.Module() + "_" + writtenState }
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket}
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
func seatBuckets() []string {
return []string{plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(),
plc.PulsesBucket(), copiesBucket()}
}
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-module-assigner's assignments, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its assignments by
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
var keptStates = []announce.State{
{Name: mma.StateAssignments, Readable: true},
{Name: mma.StateProposals, Readable: true},
{Name: mma.StateCandidates, Readable: true},
{Name: mma.StatePulses, Readable: true},
{Name: delivery.PlacementState, Readable: true},
{Name: delivery.PlacementProposalsState, Readable: true},
{Name: delivery.PlacementCandidatesState, Readable: true},
{Name: delivery.PlacementPulsesState, Readable: true},
{Name: "asks", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: assign.FindingsState, Readable: true},
{Name: buckets.PoliciesState, Readable: true},
{Name: mma.StateSeatHolders, Readable: true},
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
{Name: views.StatusState, Readable: true}, // its first views of the indexer's states (views.go)
{Name: engineReadsState, Readable: true},
{Name: retiredState}, // read only by the copy into assignments (migrate.go)
{Name: copiesState, Readable: true},
{Name: writtenState, Readable: true},
{Name: buckets.WaitsState, Readable: true},
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
{Name: health.FindingsState, Readable: true},
{Name: delivery.AssignerPoliciesState, Readable: true},
{Name: delivery.SeatHoldersState, Readable: true},
}
// kvSet is every bucket the assigner opened, by name.
type kvSet map[string]jetstream.KeyValue
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (kvSet, error) {
out := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == buckets.Puts() && err != nil {
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if slices.Contains(seatBuckets(), name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without assignments no module is placed.
if slices.Contains(seatBuckets, name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == plc.SeatHoldersBucket() && err != nil {
if name == delivery.SeatHoldersBucket && err != nil {
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
// holds which seat here, and the controller's seats still answer.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == buckets.Provisions && err != nil {
if name == provisions.ProvisionsBucket && err != nil {
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
// until then no want waits for a provision, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.PoliciesBucket() && err != nil {
if name == assign.PoliciesBucket && err != nil {
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
// person's policy is read, so the controller's beside each composition holds, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.FindingsBucket() && err != nil {
if name == assign.FindingsBucket && err != nil {
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
// router and kept as no finding.
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if name == engineReadsBucket() && err != nil {
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
// assigner_engine_reads lists nothing.
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
// a record module is asked of a person, as before.
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
continue
}
if name == buckets.Inputs() && err != nil {
// New with the verb input (design 47 G §354): only the tool process reads it, so the program starts without it.
logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err)
if name == graph.Bucket && err != nil {
// The indexer's module graph (ADR 0614 §2, §3): without it no want waits for a verb it invokes until it is bound.
logf("the bucket %s cannot be opened (%v): no want waits for a verb it invokes until it is bound once the bus says it was made", name, err)
continue
}
if name == pinsBucket() && err != nil {
// New with the pins (design 47 G §579): without it no pin is kept and no revision is removed.
logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err)
continue
}
if name == buckets.Waits() && err != nil {
// New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged
// as before.
logf("the bucket %s cannot be opened (%v): no outside wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveredAgain() && err != nil {
// New with A-250: without it assigner_delivered_again is not taken until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no version is said delivered again until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveryWaits() && err != nil {
// New with A-246: without it no dependent waits for its dependency's delivery until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no delivery wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.PutWaits() && err != nil {
// New with A-211: without it no put's wait is said until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no put's wait is said until the assigner starts again", name, err)
continue
}
if name == retiredBucket() && err != nil {
// The retired state is read only for the copy into `assignments` (migrate.go): a mesh made after the switch has
// none, and nothing is copied.
logf("the bucket %s cannot be opened (%v): nothing is copied from it", name, err)
continue
}
if name == buckets.Grants && err != nil {
if name == delivery.PartGrantsBucket && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
continue
@@ -237,11 +173,11 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket(), buckets.Provisions, assign.PoliciesBucket(), assign.FindingsBucket(),
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket():
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, graph.Bucket:
return true
}
return slices.Contains(seatBuckets(), name) || name == buckets.Grants
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
}
// run is the program: it returns when ctx ends or the bus cannot be used.
@@ -251,7 +187,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
plc.SetModule(account.Module) // the assigner names its own states from its credential's module (A-296, §428)
conn, js, err := account.Dial()
if err != nil {
return err
@@ -273,81 +208,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
findings, writer := findingsOf(ctx, account, conn, js, said, kvs, logger.Printf)
if writer != nil {
defer writer.Close()
}
// Each change of the states its readers read is said on its views family (ADR 0615 §5, ADR 0619 §2); a watch of it
// that ends stops the program, as a watch of what a pass reads does.
viewErrs := make(chan error, len(viewedStates()))
startViews(ctx, js, conn, account, kvs, wireOf(writer), viewErrs, logger.Printf)
go func() {
select {
case <-ctx.Done():
case err := <-viewErrs:
logger.Printf("%v: the assigner stops, and is started again", err)
conn.Close()
}
}()
iv := &catalogViews{js: js, account: account, call: said.Call, writer: wireOf(writer), logf: logger.Printf,
status: process.AwaitBucket(ctx, js, conn, firstViewsBucket(), "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)}
// The inputs of the composition steps it asks a build seat to run: the seat mesh-wants' verb input is answered here in chunks
// of the core connection, of any size (design 47 G §354, §712, A-237).
inputs := inputsOver(kvs[buckets.Inputs()], js, account.Module, logger.Printf)
// The seat list (seatlist.go), followed in follow over the connection and the hub: the verb input's callers are checked
// against the seat's definition it holds (A-186).
gate := seatlist.NewPart("assigner", seatListAgainCallers)
if inputs != nil {
stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)})
if err != nil {
return fmt.Errorf("answering the verb input: %w", err)
}
defer stopInput()
}
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, inputs, gate, logger, getenv)
findings := findingsOf(ctx, kvs, logger.Printf)
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, account process.Account, conn *nats.Conn, js jetstream.JetStream, said *process.Conn, kvs kvSet, logf func(string, ...any)) (assign.Findings, *heldWriter) {
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
var f assign.Findings
if kvs[assign.FindingsBucket()] == nil {
return f, nil
}
state := said.State(assign.FindingsState)
w, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
Module: account.Module, State: state, Node: account.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf})
if err != nil {
logf("the assigner's finding writer cannot be made, so its findings are not written: %v", err)
return f, nil
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
f.Put = func(x mcseat.Finding) error { return w.Write(ctx, x) }
f.Delete = func(key string) error {
scope, rest, _ := strings.Cut(key, ".")
i := strings.LastIndex(rest, ".")
if i < 0 {
return fmt.Errorf("%q is no finding's key", key)
if kv := kvs[assign.FindingsBucket]; kv != nil {
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
f.Keys = func() ([]string, error) {
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
return w.Clear(ctx, scope, rest[:i], rest[i+1:], "the assigner no longer observes it")
}
f.Keys = state.Keys
return f, w
}
// heldWriter is the assigner's finding writer: its findings are raised as they happen, so the writer holds them (design 47 G A-29).
type heldWriter = views.HeldWriter[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]
// wireOf is w as the views parts' finding writer; nil when the assigner has none, so no part writes through a nil writer.
func wireOf(w *heldWriter) views.WireWriter {
if w == nil {
return nil
}
return w
return f
}
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
@@ -358,43 +238,34 @@ type emitting struct{ *process.Conn }
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
// optionalEnds is what stops when the watch of a bucket that never gates the first pass ends.
func optionalEnds() map[string]string {
return map[string]string{buckets.Puts(): "no put is taken further", buckets.Answers: "no ask is answered"}
}
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
func watched() []string {
return []string{
buckets.Grants,
plc.AssignmentsBucket(),
plc.ProposalsBucket(),
plc.CandidatesBucket(),
plc.PulsesBucket(),
copiesBucket(),
buckets.Have,
buckets.Asks(),
buckets.Differences(),
buckets.Wants(),
assign.PoliciesBucket(),
buckets.Provisions,
plc.SeatHoldersBucket(),
engineReadsBucket(),
writtenBucket(),
buckets.Waits(),
buckets.PutWaits(),
buckets.DeliveryWaits(),
buckets.DeliveredAgain(), // its watch wakes a pass at once on an ask of assigner_delivered_again (A-250)
}
var watched = []string{
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
assign.PoliciesBucket,
provisions.ProvisionsBucket,
delivery.SeatHoldersBucket,
delivery.SourceBucket,
delivery.MergesBucket,
graph.Bucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, inputs *inputKeeper, gate *seatlist.Part, logger *log.Logger, getenv func(string) string) error {
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, logger *log.Logger) error {
w := newWorld()
// The callers' seat entries are resolved from the assigner's own seat-holders as it last read them (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return w.read(plc.SeatHoldersBucket()), nil }))
dirty := make(chan struct{}, 1)
wake := func() {
select {
@@ -402,26 +273,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
default:
}
}
errs := make(chan error, len(watched())+2)
settled := make(chan struct{}, len(watched())+2)
// What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read
// from the node's record where the engine does not answer (the fallback, said in the journal).
reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf}
if bus != nil {
reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1)
reads.Once = bus.CallOn // its fault keeps the fallback at once while the kept call asks on (ADR 0668 §3)
}
if kv := kvs[engineReadsBucket()]; kv != nil {
reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
reads.Delete = func(node string) error { return kv.Delete(ctx, node) }
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
watch := func(name string, opts ...jetstream.WatchOpt) error {
kw, err := kvs[name].WatchAll(ctx, opts...)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == buckets.Puts() || name == buckets.Answers
optional := name == delivery.AssignerPutsBucket
go func() {
defer kw.Stop()
for {
@@ -432,7 +292,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case e, ok := <-kw.Updates():
if !ok {
if optional {
logger.Printf("the watch of %s ended: %s until the assigner starts again", name, optionalEnds()[name])
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
return
}
errs <- fmt.Errorf("the watch of %s ended", name)
@@ -448,10 +308,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
if name == engineReadsBucket() {
reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed
continue
}
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
wake()
}
@@ -461,10 +317,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
if kvs[buckets.Puts()] != nil {
if err := watch(buckets.Puts()); err != nil {
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, buckets.Puts())
delete(kvs, delivery.AssignerPutsBucket)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
@@ -473,75 +329,42 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[buckets.Puts()] != nil || time.Since(triedPuts) < PutsReopenEvery {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, buckets.Puts())
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
if err != nil {
return
}
kvs[buckets.Puts()] = kv
if err := watch(buckets.Puts()); err != nil {
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, buckets.Puts())
delete(kvs, delivery.AssignerPutsBucket)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", buckets.Puts())
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
started := 0
for _, name := range watched() {
for _, name := range watched {
if kvs[name] == nil && startsWithout(name) {
continue // not on the bus at this start (open said so)
}
started++
var opts []jetstream.WatchOpt
if name == buckets.Have {
// what each node ran healthy before, for a set-back
if name == delivery.HaveBucket || name == delivery.SourceBucket {
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
opts = append(opts, jetstream.IncludeHistory())
}
if err := watch(name, opts...); err != nil {
return err
}
}
// The assigner's one pull of its durable on the shared events stream: the indexer's views and the engines' reads changed
// are handlers of it, started once both are registered (process.EventHub).
hub := process.NewEventHub(logger.Printf)
// The seat list (seatlist.go), read over the connection and heard on the hub, given to catalogue at every change, and a
// pass made then: registered before the hub starts.
if bus != nil {
gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
}
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
// §389). Registered before the hub starts.
ends, away := &runEnds{}, &awayNodes{}
if inputs != nil {
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
return err
}
}
for subject, isAway := range map[string]bool{meshwatchdogs.SubjectMachineAwayAll: true, meshwatchdogs.SubjectMachineBackAll: false} {
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPresence(away, isAway, m, logger.Printf) }); err != nil {
logger.Printf("%s is not heard, so a composition step asks a machine that sleeps: %v", subject, err)
}
}
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
// (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands.
if iv != nil {
iv.hub = hub
started += iv.start(ctx, w, wake, settled)
// The teller's answers' picture is watched as a bucket is, but never gates the first pass: a teller that does not
// answer its first view only leaves the asks unanswered, said by the reader's finding (A-175).
if iv.answers != nil {
kvs[buckets.Answers] = iv.answers
if err := watch(buckets.Answers); err != nil {
logger.Printf("%v: no ask is answered until the assigner starts again", err)
}
}
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
@@ -550,34 +373,30 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
name string
kv jetstream.KeyValue
}
assignmentsAppeared := make(chan appeared, len(seatBuckets())+2)
for _, name := range seatBuckets() {
placementAppeared := make(chan appeared, len(seatBuckets)+2)
for _, name := range seatBuckets {
if kvs[name] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-module-assigner waits for it",
logger.Printf).Ready(func(kv jetstream.KeyValue) { assignmentsAppeared <- appeared{name, kv} })
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), buckets.Provisions, plc.SeatHoldersBucket()} {
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, delivery.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { assignmentsAppeared <- appeared{name, kv} })
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
}
takeAssignments := func(a appeared) error {
takePlacement := func(a appeared) error {
if kvs[a.name] != nil {
return nil // bound already, by the other of its two events
}
kvs[a.name] = a.kv
logger.Printf("the bucket %s is open and watched", a.name)
if err := watch(a.name); err != nil {
return err
}
wake() // a state made empty brings no change of its own: the pass copies the retired state into it (migrate.go)
return nil
return watch(a.name)
}
for range started {
for range started + 1 {
select {
case <-ctx.Done():
return nil
@@ -586,109 +405,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case <-settled:
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
reads: reads, seats: gate}
// The wait on an outside cause (outside.go, design 47 G §706): the keeper's three events say when to judge again, and the
// seat mesh-wants' wait-held and wait-ended are said; registered before the hub starts.
if bus != nil && kvs[buckets.Waits()] != nil {
bound, settle, err := outsideSettings(getenv)
if err != nil {
return err
}
p.outside = newOutsideActs(bus.CallKept, wake, logger.Printf, bound, settle)
for _, subject := range []string{mcseat.SubjectConditionRaised, mcseat.SubjectConditionChanged, mcseat.SubjectConditionCleared} {
subject := subject
if err := hub.Handle(subject, func(m jetstream.Msg) {
p.outside.Heard(m.Subject(), m.Data())
if err := m.Ack(); err != nil {
logger.Printf("acknowledging %s: %v", m.Subject(), err)
}
}); err != nil {
logger.Printf("%s is not heard, so a wait on an outside cause is judged again only when its check changes: %v", subject, err)
}
}
p.outside.readStart()
}
if js != nil {
p.publishWants = wantsPublisher(ctx, js, thisNode)
if iv != nil && iv.h != nil {
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)
p.graph = &graphKeeper{objects: objects, halves: iv.h.views}
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
return p.keepCandidate(ctx, iv.call, ev)
}
iv.candidates.Store(&keep)
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
p.entries = meshcatalogclient.NewEntries(iv.h.call)
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
// on the hub, registered before it starts.
if kv := kvs[pinsBucket()]; kv != nil {
k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf}
if err := k.made(); err != nil {
logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err)
}
if err := hearPins(hub, k); err != nil {
logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err)
} else {
p.graph.pins = k
k.watchUnpins(kv)
}
}
}
}
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
// Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it
// is not there yet, a node is asked again only when it comes back online.
if js != nil {
// The teller's ask-ended (issue 1966): an ask ended at the channel without an answer is written again; registered
// before hearReadsChanged starts the hub.
if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks()], staleOfWorld(w), logger.Printf); err != nil {
logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err)
}
readEndedAtBind(ctx, js, hub, kvs[buckets.Asks()], staleOfWorld(w), bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf)
if kv := kvs[writtenBucket()]; kv != nil {
// The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow
// once the first registrations are written (§95, §353).
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
p.ends.trunk = w.trunkName
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
err := hearEnds(hub, p.ends)
if err == nil && bus != nil {
err = followRegistrations(ctx, hub, bus.Call, p.ends)
}
if err != nil {
logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
}
}
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
if err := hearTaken(hub, &w.taken, wake, logger.Printf); err != nil {
logger.Printf("the indexer's take-ins are not heard, so an entry held by an order edge waits until the assigner starts again: %v", err)
}
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
hearRunning(ctx, js, w, wake, findings, logger.Printf)
}
if bus != nil && kvs[buckets.Puts()] != nil {
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, call: bus.CallKeptOn}, keeper: inputs, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep)
}
}
if bus != nil {
p.seat = newSeatActs(bus, wake, logger.Printf)
}
if js != nil {
p.publish = seatPublisher(ctx, js, thisNode)
p.publish = streamPublisher(ctx, js, thisNode)
}
wake()
for {
@@ -698,8 +423,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case err := <-errs:
return err
case <-settled:
case kv := <-assignmentsAppeared:
if err := takeAssignments(kv); err != nil {
case kv := <-placementAppeared:
if err := takePlacement(kv); err != nil {
return err
}
case <-dirty:
@@ -708,12 +433,12 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
// bus now holds, and says the seat waits while one is missing.
var unbound []string
for _, name := range seatBuckets() {
for _, name := range seatBuckets {
if kvs[name] != nil || js == nil {
continue
}
if kv, err := js.KeyValue(ctx, name); err == nil {
if err := takeAssignments(appeared{name, kv}); err != nil {
if err := takePlacement(appeared{name, kv}); err != nil {
return err
}
continue
@@ -731,13 +456,13 @@ var PutsReopenEvery = time.Minute
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) error {
return func(c oc.Condition) error {
subject, err := occ.ConditionSubject(assignment.Module())
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(delivery.AssignerModule)
if err != nil {
return err
}
data, err := occ.EncodeCondition(c)
data, err := asks.EncodeCondition(c)
if err != nil {
return err
}
@@ -745,54 +470,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) erro
return err
}
}
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: acknowledges it, keeps what it says
// of the node (a statement older than it does not stand) and asks the node again.
func readsChangedHandler(reads *enginereads.Keeper, logf func(string, ...any)) func(jetstream.Msg) {
return func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
switch {
case !ours:
case err != nil:
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
reads.AskNode(node, enginereads.Event+" that does not read")
default:
// What the event says is kept first: a statement older than it does not stand while the node is asked again.
if e, derr := enginereads.DecodeChanged(m.Data()); derr == nil {
reads.HeardChanged(e)
}
reads.AskNode(node, enginereads.Event)
}
}
}
// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's one hub of its durable
// consumer on EVENTS, and asks the node it names again; the hub also hands the indexer's views to the followers of
// indexerViews, and starts once both are registered.
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *enginereads.Keeper, findings assign.Findings, live func(), logf func(string, ...any)) {
account := process.Account{Node: thisNode, Module: assignment.Module()}
w := notHeardWriter(findings, logf)
handle := readsChangedHandler(reads, logf)
err := hub.Handle(enginereads.SubjectPrefix+"*", handle)
// At each bind every node the assigner holds a statement of is asked again: an event said before its consumer existed
// is not delivered to it (issue 1966 note 155: ace's bootstrapped statement stood; ADR 0668 §4).
hub.OnBound(reads.AtBind(func() []string { return slices.Sorted(maps.Keys(reads.Statements())) }))
if err == nil {
err = hub.Start(ctx, js, account)
}
if err != nil {
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again (a node is asked again when it comes back online)", err)
return
}
w.Heard(process.EventsStream)
if live != nil {
live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive)
}
go func() {
<-ctx.Done()
hub.Stop()
}()
}
@@ -1,12 +1,7 @@
package main
import (
"encoding/json"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"context"
"errors"
@@ -23,21 +18,14 @@ import (
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
if viewsEmitter(nil, process.Account{}, nil, &heldWriter{}, t.Logf).Findings == nil {
t.Error("the assigner's views emitter writes no view-unsaid finding")
}
// 1297/B2a: without a credential the program does not start, and says why.
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
@@ -69,7 +57,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, pair := range []struct {
what string
code, want []string
}{{"reads", readBuckets, reads}, {"states", append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()), states}} { // the marks are bound by the views family (views.go)
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
if !reflect.DeepEqual(got, want) {
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
@@ -79,7 +67,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, s := range keptStates {
kept = append(kept, m.Module+"_"+s.Name)
}
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()))); !reflect.DeepEqual(got, want) {
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
t.Errorf("the program announces %v, keeps %v", got, want)
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
@@ -90,17 +78,17 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
starts bool
said string
}{
{buckets.Puts(), jetstream.ErrBucketNotFound, true, "no put is taken further"},
{buckets.Puts(), errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{buckets.Puts(), context.DeadlineExceeded, true, "no put is taken further"},
{buckets.Wants(), errors.New("nats: permissions violation"), false, ""},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets())-1 || !strings.Contains(logged, c.said)):
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
@@ -111,21 +99,21 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
placed, err := mmac.AssignmentsValues(mma.Assignments{Node: "ace"})
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
if err != nil {
t.Fatal(err)
}
for key, raw := range placed {
kv := kvs[plc.AssignmentsBucket()].(*watchKV)
kv := kvs[delivery.PlacementBucket].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[buckets.Puts()].(*watchKV).closed = true
kvs[buckets.Wants()].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner", seatListAgainCallers), log.New(io.Discard, "", 0), func(string) string { return "" })
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -195,17 +183,3 @@ func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (e kvEntry) Revision() uint64 { return 1 }
func (e kvEntry) Created() time.Time { return time.Time{} }
// Keys lists nothing: the retired state this watch stands for holds no key.
func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
return nil, jetstream.ErrNoKeysFound
}
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
return err
}
}
@@ -1,150 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
plc "mesh-delivery-assigner/internal/assignment"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
)
// The state this seat's assignments were kept in before ADR 0641 gave it its name. The assigner reads it here and nowhere
// else: at its start it copies each key into `assignments` (copyRetired) and keeps no reader of it afterwards. The
// manifest still declares the state, so the assigner's account may read it for the copy; a later release drops it.
const retiredState = "placement"
func retiredBucket() string { return plc.Module() + "_" + retiredState }
// CopiedEvent is the module's event saying the copy was made, once: mesh.mod.mesh-delivery-assigner.event.assignments-copied.
const CopiedEvent = "assignments-copied"
// copiedSaid is what the event carries: the keys copied, those already in `assignments` (kept as they are), and the keys of
// the form `placement.<node>` that were dropped (their node key holds the value).
type copiedSaid struct {
Copied int `json:"copied"`
Present int `json:"present"`
Dropped int `json:"dropped"`
Skipped int `json:"skipped"`
}
// copiesState is the assigner's own state of its copies, read by its tool assigner_read_copies, and copiesBucket its bucket.
// The retired state gets nothing written: readers of other modules still parse every key in it.
const copiesState = "copies"
func copiesBucket() string { return plc.Module() + "_" + copiesState }
// copiedMarker is the key, in the state copies, that says the copy ended: its time and the keys it copied. It is written
// after the last key, and the copy is made at every start until it is there and never after, so a copy cut short by a
// crash is completed at the next start and a key `assignments` dropped after the copy ended is never brought back.
const copiedMarker = "copied-from-retired"
type copiedMark struct {
At time.Time `json:"at"`
Copied int `json:"copied"`
}
// copyRetired is the holder's switch from the retired state to `assignments`, with no gap: until the marker is there, it
// creates in `assignments` each key of the retired state in the form `<node>.node` or `<node>.<module>` that is missing
// there (never overwriting one), the module keys first and the node keys, which commit a node, last, and then writes the
// marker. It deletes nothing: the readers delivered before this one still read the retired state, and a delete of a
// `<node>.node` key there is a node leaving (ADR 0641 section 4); the state is dropped in the next release. Keys of the
// form `placement.<node>` are not copied, and a key that fits no form is left and counted. It answers whether the seat may
// go on: a copy that failed is tried again at the next pass.
func (p *passer) copyRetired(ctx context.Context) bool {
if p.copied {
return true
}
old, next, copies := p.kvs[retiredBucket()], p.kvs[plc.AssignmentsBucket()], p.kvs[copiesBucket()]
if next == nil || copies == nil {
return false
}
if old == nil {
p.logf("assignments: the retired state %s cannot be read, so nothing is copied from it", retiredBucket())
p.copied = true
return true
}
switch _, err := copies.Get(ctx, copiedMarker); {
case err == nil:
p.copied = true // the copy ended at an earlier start: nothing is copied again
return true
case !errors.Is(err, jetstream.ErrKeyNotFound):
p.logf("assignments: the marker %s of %s cannot be read, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
names, err := old.Keys(ctx)
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
p.logf("assignments: the keys of %s cannot be listed, and are tried again at the next pass: %v", retiredBucket(), err)
return false
}
// A node's module keys before its node key: the node key is the commit the readers wait for.
slices.SortStableFunc(names, func(a, b string) int {
return strings.Compare(sortForm(a), sortForm(b))
})
said := copiedSaid{}
for _, key := range names {
_, _, kind, perr := mmac.ParseKey(key)
switch {
case strings.HasPrefix(key, retiredState+"."):
// The node named placement could not exist before, so this is the form that held a whole node in one key.
said.Dropped++
case perr != nil || (kind != mmac.NodeKind && kind != mmac.ModuleKind):
said.Skipped++
continue
default:
entry, err := old.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
p.logf("assignments: %s cannot be read, and the copy is tried again at the next pass: %v", key, err)
return false
}
if entry.Operation() != jetstream.KeyValuePut {
continue
}
rev, err := next.Create(ctx, key, entry.Value())
switch {
case errors.Is(err, jetstream.ErrKeyExists):
said.Present++
case err != nil:
p.logf("assignments: %s is not copied, and the copy is tried again at the next pass: %v", key, err)
return false
default:
said.Copied++
p.w.putEntry(plc.AssignmentsBucket(), key, entry.Value(), rev, p.clock())
}
}
}
mark, _ := json.Marshal(copiedMark{At: p.clock(), Copied: said.Copied + said.Present})
if _, err := copies.Put(ctx, copiedMarker, mark); err != nil {
p.logf("assignments: the end of the copy is not written as %s in %s, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
p.copied = true
if said == (copiedSaid{}) {
return true // the retired state holds nothing
}
p.logf("assignments: copied %d keys of %s into %s (%d already there kept as they are, %d of the form placement.<node> not copied, %d fitting no form left)",
said.Copied, retiredBucket(), plc.AssignmentsBucket(), said.Present, said.Dropped, said.Skipped)
if p.seat != nil {
if err := p.seat.bus.Emit(CopiedEvent, said); err != nil {
p.logf("assignments: the copy is not said as an event: %v", err)
}
}
return true
}
// sortForm orders `<node>.<module>` keys before `<node>.node`.
func sortForm(key string) string {
if node, ok := strings.CutSuffix(key, ".node"); ok {
return node + "\x7f"
}
return key
}
@@ -1,118 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assigngraph"
)
// busObjects is the assigner's own object bucket assignment-revisions in the old `objects` section (novox/hq ADR 0619 §2),
// which the grants part creates, as the passer's process reaches it: opened at its first use and kept; until the bucket exists
// every call is an error saying so, and the next call opens it again (design 47 G A-67, modelled on the indexer's busObjects).
type busObjects struct {
js jetstream.JetStream
module string
local string
mu sync.Mutex
o jetstream.ObjectStore
}
// newBusObjects is the object bucket of this local name of the module the account runs as.
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
return &busObjects{js: js, module: cred.Module, local: local}
}
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.o != nil {
return b.o, nil
}
o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(b.module, b.local))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
}
b.o = o
return o, nil
}
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
o, err := b.open(ctx)
if err != nil {
return err
}
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
return err
}
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
data, err := o.GetBytes(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, assigngraph.ErrNoObject
}
return data, err
}
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
infos, err := o.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return []string{}, nil
}
if err != nil {
return nil, err
}
names := make([]string, 0, len(infos))
for _, i := range infos {
names = append(names, i.Name)
}
return names, nil
}
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
func (b *busObjects) Delete(ctx context.Context, name string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
o, err := b.open(ctx)
if err != nil {
return 0, 0, err
}
st, err := o.Status(ctx)
if err != nil {
return 0, 0, err
}
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
if err != nil {
return int64(st.Size()), 0, err
}
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
}
@@ -1,296 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The three order edges of a delivery that crosses a form change (design 47 G §298, A-245, A-258, A-259, A-260). Each is an acting wait of the
// assigner, kept in the state `waits` under `order.<node>.<module>.<part>` beside the acting waits, in the step placing, said as wait-held
// when a want is held for it and wait-ended (ran, released, replaced) when it moves; assigner_waits reads them and assigner_release with
// how `now` releases them, as it does an acting wait. The holders of the seats are read from the seat holders the assignments compose
// (assignment.SeatHoldersOf), never named:
// - the node-backup holder's and the bus module's (the holder of the seat mesh-broker) new versions run before a module whose kept data's
// form changes is moved, since the restore point needs both (OrderBackup);
// - at a revert a node's node-engine goes back last on that node, after every module it restores there runs the reverted build, which it
// starts only once it has said `restored` (OrderEngineLast);
// - the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a source revision at least
// the trunk's head, the highest `src` of the trunk branch's module values (OrderTakenIn, A-260).
// takenSubject is the indexer's event take-in.taken on its own stream, as the manifest's consumes names it.
var takenSubject = process.ModuleStreamSubject(buckets.IndexerModule, "take-in.taken")
// takenSources are the newest source revisions a take-in.taken named, by watched branch, as heard (hearBuildsAndReleases).
type takenSources struct {
mu sync.Mutex
own map[string]int64
}
// put takes one take-in.taken in. A hand drop, and a taken with no source revision (one said before G), take nothing in.
func (t *takenSources) put(data []byte) error {
var e meshcatalog.TakeInTaken
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a take-in.taken that is none: %w", err)
}
if e.Dropped || e.Source == 0 || e.Branch == "" {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.own == nil {
t.own = map[string]int64{}
}
t.own[e.Branch] = max(t.own[e.Branch], e.Source)
return nil
}
// of is the newest source revision of branch taken in; 0 before one is heard.
func (t *takenSources) of(branch string) int64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.own[branch]
}
// orderEdge is one order edge held on module's want on node: part's version fp must run on partNode (OrderBackup, OrderEngineLast), or the
// trunk's source revision source be taken in (OrderTakenIn).
type orderEdge struct {
node, module, part, partNode, fp, entry, order string
source int64
}
func (e orderEdge) key() string { return acting.OrderKey(e.node, e.module, e.part) }
func (e orderEdge) acting() assign.ActingEdge {
return assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace}
}
// orderRuns says an order edge is met: its part runs the version, or the source revision is taken in.
func (p *passer) orderRuns(n assign.Nodes, e orderEdge) bool {
if e.order == acting.OrderTakenIn {
return p.w.taken.of(p.w.trunkName()) >= e.source
}
return n.Runs(e.acting())
}
// seatHolders are the holders of each seat as the assignments compose them, by seat name.
func (p *passer) holdersBySeat() map[string][]mma.SeatHoldersHoldersItem {
out := map[string][]mma.SeatHoldersHoldersItem{}
for _, sh := range assignment.SeatHoldersOf(p.w.assignments()) {
out[sh.Seat] = append(out[sh.Seat], sh.Holders...)
}
return out
}
// orderEdgesOf computes §298's edges on the wants of module m on each of its nodes, into p.orders. wanted is m's wants by node; trunk the
// trunk's module values. A node that has no running build heard, runs m at its trunk version already, or is wanted that version already
// has none: nothing moves. It answers, by node, whether an edge still stands that cannot be recorded as a wait because the catalog entry
// it reads is not answered yet (asked, and its answer wakes a pass): the want is held meanwhile, never written across a form change unread.
func (p *passer) orderEdgesOf(ctx context.Context, m assign.Module, wanted map[string]mwc.Want, trunk map[string]meshcatalog.ModulesByModuleValue) map[string]bool {
unread := map[string]bool{}
p.orderVisited[m.Name] = true
if m.FP == "" || p.graph == nil || p.entries == nil {
return unread
}
var holders map[string][]mma.SeatHoldersHoldersItem
for _, node := range m.Nodes {
if w, ok := wanted[node]; ok && !w.Removed && w.Fp == m.FP {
continue // the version is wanted already: nothing moves, so nothing waits
}
run, heard := p.w.runningOf(node, m.Name)
if !heard || run.Stopped || run.Fingerprint == "" || run.Fingerprint == m.FP {
continue
}
if holders == nil {
holders = p.holdersBySeat()
}
held := func(seatName string) bool {
for _, h := range holders[seatName] {
if h.Module == m.Name && h.Node == node {
return true
}
}
return false
}
add := func(e orderEdge) {
e.node, e.module, e.entry = node, m.Name, m.FP
p.orders[e.key()] = e
}
// 1. a kept data's form changes: the restore point needs the new node-backup holder and bus module.
changes, known := p.formChange(ctx, m.Name, run.Fingerprint, m.FP)
if !known {
unread[node] = true
}
if len(changes) > 0 {
for seatName, onNode := range map[string]bool{catalogue.BackupSeat: true, meshbroker.Seat: false} {
for _, h := range holders[seatName] {
if h.Module == m.Name || (onNode && h.Node != node) || trunk[h.Module].Fp2 == "" {
continue
}
add(orderEdge{part: h.Module, partNode: h.Node, fp: trunk[h.Module].Fp2, order: acting.OrderBackup})
}
}
}
if !held(nodeengine.Seat) && !held(mma.Seat) && !held(meshdelivery.Seat) {
continue
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, m.Name, run.Fingerprint, m.FP)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", m.Name, node, m.FP, err)
continue
}
if !reverted {
continue
}
// 2. at a revert the node-engine goes back last on its node: after every module that is set back there across a form change runs.
if held(nodeengine.Seat) {
for module, byNode := range p.w.wantsOf() {
// What it is set back to is its trunk version, which the rules write as its want in this pass.
target := trunk[module].Fp2
if w, ok := byNode[node]; module == m.Name || !ok || w.Removed || target == "" {
continue
}
xrun, heard := p.w.runningOf(node, module)
if !heard || xrun.Stopped || xrun.Fingerprint == "" || xrun.Fingerprint == target {
continue
}
if back, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, xrun.Fingerprint, target); err != nil || !back {
continue
}
xchanges, known := p.formChange(ctx, module, xrun.Fingerprint, target)
if !known {
unread[node] = true
}
if len(xchanges) > 0 {
add(orderEdge{part: module, partNode: node, fp: target, order: acting.OrderEngineLast})
}
}
}
// 3. the reverted assigner and gate wait for the indexer to take in the trunk's head.
if held(mma.Seat) || held(meshdelivery.Seat) {
var head int64
for _, v := range p.w.trunkOf(p.w.trunkName()) {
head = max(head, v.Src)
}
if head > 0 {
add(orderEdge{part: buckets.IndexerModule, fp: strconv.FormatInt(head, 10), source: head, order: acting.OrderTakenIn})
}
}
}
return unread
}
// orderActing is the order edges of module on node that still hold its want, as acting edges: those not met, and not released by hand. A
// node whose catalog entry is not read yet holds its want too, on a stand-in edge no part runs.
func (p *passer) orderActing(module string, released map[string]bool, unread map[string]bool, n func() assign.Nodes) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
nodes := n()
for _, e := range p.orders {
if e.module == module && e.node == node && !released[e.key()+"@"+e.fp] && !p.orderRuns(nodes, e) {
out = append(out, assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace})
}
}
if unread[node] {
out = append(out, assign.ActingEdge{Node: node, Part: "the catalog entry", Step: seatdef.StepPlace})
}
return out
}
}
// orderWaits keeps the order waits after the pass's wants, as actingWaits keeps the acting waits: it starts the wait of each edge not met,
// ends the wait of one that ran, was replaced or was released, and says each.
func (p *passer) orderWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]mwc.Want) {
if p.kvs[buckets.Waits()] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits())
trunk := p.w.trunk()
if p.orderSaid == nil {
p.orderSaid = map[string]bool{}
}
standing := map[string]bool{}
for key, e := range p.orders {
if w, ok := wanted[e.module][e.node]; ok && !w.Removed && w.Fp == e.entry {
continue
}
if p.orderRuns(n, e) {
continue
}
standing[key] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == e.entry && w.Fp == e.fp {
if !w.Released && !p.orderSaid[key] {
// A wait still held after a restart is said again, as wait-held (A-260).
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry+".again"); err != nil {
p.logf("the wait of %s on %s for %s is not said again, and is said at the next pass: %v", e.module, e.node, e.part, err)
} else {
p.orderSaid[key] = true
}
}
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[e.node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: seatdef.StepPlace, Node: e.node, Module: e.module, Part: e.part, PartNode: e.partNode, Fp: e.fp, Entry: e.entry, Order: e.order,
Since: now, ClockFrom: now, Bound: acting.BoundOf(seat.Edge{Step: seatdef.StepPlace}).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", e.module, e.node, e.part, err)
continue
}
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
p.orderSaid[key] = true
}
for key, raw := range kept {
if !acting.IsOrder(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
if _, there := trunk[w.Module]; there && !p.orderVisited[w.Module] {
continue // the module was not looked at in this pass (no pipeline passed for it yet): its wait stands as kept
}
how := acting.HowRan
switch e, there := p.orders[key]; {
case w.Released:
how = acting.HowReleased
case there && e.fp != w.Fp, !there && wanted[w.Module][w.Node].Fp != w.Entry:
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module())
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits(), key)
delete(p.orderSaid, key)
}
}
@@ -1,351 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
func orderFP(c string) string {
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
}
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
type orderMod struct {
name string
run, trunk string
forms map[string]int
holds []string
src int64
}
type orderFixture struct {
t *testing.T
p *passer
wants *memKV
events []string
bodies []meshwants.Wait
answers chan struct{}
}
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
t.Helper()
ctx := context.Background()
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
p := f.p
p.logf = t.Logf
p.say = func(oc.Condition) error { return nil }
p.publishWants = func(event string, body any, id string) error {
f.events = append(f.events, event)
f.bodies = append(f.bodies, body.(meshwants.Wait))
return nil
}
digest := "sha256:" + strings.Repeat("c", 64)
pl := mma.Assignments{Node: "ace"}
forms := map[string]int{} // by fingerprint: each module's own
modules := []string{}
for _, m := range mods {
modules = append(modules, m.name)
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
for _, seat := range m.holds {
scope := "node"
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
scope = "mesh"
}
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
}
for fp, form := range m.forms {
forms[fp] = form
}
mdigest := fmt.Sprintf("%064x", len(modules))
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, Src: m.src})
putModuleRaw(p.w, trunkBranch, m.name, mod)
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
endDone(t, p.w, m.name, m.trunk, mdigest)
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
f.runs(m.name, m.run)
wkey, _ := mwc.WantKey("ace", m.name)
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
p.w.put(buckets.Wants(), wkey, old)
f.wants.values[wkey] = old
}
composes(t, p.w, "ace", digest)
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
objects := memObjects{}
for i, at := range history {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, module := range modules {
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
}
placed := mma.Assignments{Node: "ace"}
for _, module := range modules {
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]mma.Assignments{"ace": placed}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
if form := forms[in.Fingerprint]; form > 0 {
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
return f
}
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
func (f *orderFixture) runs(module, fp string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
f.p.w.put(buckets.Have, "ace."+module, have)
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
}
// pass makes a pass, and after it every catalog entry it asked is answered.
func (f *orderFixture) pass() {
f.t.Helper()
f.p.wants(context.Background())
asked := 0
for _, row := range f.p.entries.Rows() {
if row.State == meshcatalogclient.EntryAsked {
asked++
}
}
for range asked {
select {
case <-f.answers:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
f.t.Fatal("an entry asked was not answered")
}
}
}
// wantAt is the fingerprint module is wanted at on ace.
func (f *orderFixture) wantAt(module string) string {
f.t.Helper()
key, _ := mwc.WantKey("ace", module)
w, err := mwc.DecodeWant(f.wants.values[key])
if err != nil {
f.t.Fatal(err)
}
return w.Fp
}
func (f *orderFixture) waits() []string {
var out []string
for key := range f.p.w.read(buckets.Waits()) {
if acting.IsOrder(key) {
out = append(out, key)
}
}
return out
}
// Design 47 G §298 first edge, A-258, A-259: a module whose kept data's form changes is not moved until the new version of the node-backup
// holder and of the holder of the seat mesh-broker run; the wait is kept under `order.` in the state waits, said as wait-held, and ends
// ran (wait-ended) when the holders run; a module whose kept data keeps its form is moved at once.
func TestAModuleWhoseKeptDataChangesFormWaitsForTheNewBackupHolderAndTheBusModule(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "plain", run: orderFP("c"), trunk: orderFP("d"), forms: map[string]int{orderFP("c"): 1, orderFP("d"): 1}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
})
f.pass() // the entries are asked: no want crosses a form change unread
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web is wanted at %s before the entries are read", got)
}
f.pass()
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web moved across a form change at %s while the new node-backup holder and bus module do not run", got)
}
if got := f.wantAt("plain"); got != orderFP("d") {
t.Errorf("a module whose kept data keeps its form is wanted at %s, not moved", got)
}
if len(f.waits()) != 2 || len(f.events) != 2 || f.events[0] != meshwants.EventWaitHeld || f.bodies[0].Step != seatdef.StepPlace || f.bodies[0].Module != "web" {
t.Fatalf("waits %v events %v bodies %+v: want two wait-held in the step place, `order.ace.web.backup` and `order.ace.web.bus`", f.waits(), f.events, f.bodies)
}
if _, err := acting.Decode(f.p.w.read(buckets.Waits())[acting.OrderKey("ace", "web", "backup")]); err != nil {
t.Fatalf("the wait is not kept under its key: %v", err)
}
f.pass()
if len(f.events) != 2 {
t.Fatalf("a pass that changed nothing said %v", f.events)
}
f.runs("backup", orderFP("f"))
f.pass()
if got := f.wantAt("web"); got != orderFP("a") || len(f.events) != 3 || f.events[2] != meshwants.EventWaitEnded || f.bodies[2].How != acting.HowRan {
t.Fatalf("with the bus module still old: web at %s, events %v", got, f.events)
}
f.runs("bus", orderFP("h"))
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 4 {
t.Fatalf("with both running: web at %s, waits %v, events %v", got, f.waits(), f.events)
}
}
// §298 first edge, the hand: assigner_waits reads an order wait and assigner_release with how `now` lets the module move.
func TestAnOrderWaitIsReadAndReleasedByTheExistingTools(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
})
f.pass()
f.pass()
var acts []string
s := newPutTools(memStore{waitsState: f.p.kvs[buckets.Waits()].(*memKV), buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
got, err := s.doWaits(map[string]any{"module": "web"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).Key != "order.ace.web.backup" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "web", "how": "now", "why": "the restore point is taken by hand"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
f.p.w.put(buckets.Waits(), "order.ace.web.backup", f.p.kvs[buckets.Waits()].(*memKV).values["order.ace.web.backup"])
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || f.bodies[len(f.bodies)-1].How != acting.HowReleased {
t.Fatalf("after the release: web at %s, last %+v", got, f.bodies[len(f.bodies)-1])
}
}
// §298 second edge, A-259: at a revert the node-engine goes back last on its node, after every module that is set back there across a
// form change runs the reverted build.
func TestAtARevertTheNodeEngineGoesBackLast(t *testing.T) {
old, now := map[string]string{"web": orderFP("a"), "engine": orderFP("c")}, map[string]string{"web": orderFP("b"), "engine": orderFP("d")}
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("b"), trunk: orderFP("a"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "engine", run: orderFP("d"), trunk: orderFP("c"), holds: []string{nodeengine.Seat}},
}, old, now)
f.pass()
f.pass()
if f.wantAt("web") != orderFP("a") {
t.Fatalf("web is wanted at %s: the module set back is not held", f.wantAt("web"))
}
if got := f.wantAt("engine"); got != orderFP("d") {
t.Fatalf("the node-engine is set back to %s before web runs its reverted build", got)
}
if w := f.waits(); len(w) != 1 || w[0] != "order.ace.engine.web" || f.bodies[len(f.bodies)-1].Module != "engine" {
t.Fatalf("waits %v bodies %+v", w, f.bodies)
}
f.runs("web", orderFP("a"))
f.pass()
f.pass()
if got := f.wantAt("engine"); got != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("with web back: the node-engine is wanted at %s, waits %v", got, f.waits())
}
}
// §298 third edge, A-259, A-260: the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a
// source revision at least the trunk's head (the highest src of the trunk's module values).
func TestTheRevertedAssignerAndGateWaitForTheIndexerToTakeInTheTrunksHead(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a"), "gate": orderFP("c"), "idx": orderFP("e")}, map[string]string{"asg": orderFP("b"), "gate": orderFP("d"), "idx": orderFP("f")}
f := newOrderFixture(t, []orderMod{
{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 6},
{name: "gate", run: orderFP("d"), trunk: orderFP("c"), holds: []string{meshdelivery.Seat}, src: 7},
{name: "idx", run: orderFP("e"), trunk: orderFP("e"), src: 7},
}, old, now)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("b") || f.wantAt("gate") != orderFP("d") {
t.Fatalf("the reverted assigner (%s) and gate (%s) are wanted before the head is taken in", f.wantAt("asg"), f.wantAt("gate"))
}
if len(f.waits()) != 2 || f.bodies[0].Part != "mesh-delivery-indexer" {
t.Fatalf("waits %v bodies %+v", f.waits(), f.bodies)
}
taken := func(branch string, source int64) {
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: branch, Source: source})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
}
taken("beta", 9) // another branch, and
taken(trunkBranch, 6)
f.pass()
if f.wantAt("asg") != orderFP("b") {
t.Fatal("a source revision before the trunk's head was taken in, and the assigner was wanted")
}
taken(trunkBranch, 7)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || f.wantAt("gate") != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("after the head is taken in: asg %s gate %s waits %v", f.wantAt("asg"), f.wantAt("gate"), f.waits())
}
}
// A-260: an order wait still held after a restart is said again as wait-held, by a pass that remembers nothing, and ends at the next
// take-in.taken that names the trunk's head.
func TestAnOrderWaitHeldAfterARestartIsSaidAgain(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a")}, map[string]string{"asg": orderFP("b")}
f := newOrderFixture(t, []orderMod{{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 7}}, old, now)
f.pass()
f.pass()
if len(f.events) != 1 {
t.Fatalf("events %v", f.events)
}
f.p.orderSaid = nil // a process started again
f.pass()
if len(f.events) != 2 || f.events[1] != meshwants.EventWaitHeld || len(f.waits()) != 1 {
t.Fatalf("events %v waits %v", f.events, f.waits())
}
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: trunkBranch, Source: 8})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || len(f.waits()) != 0 || f.events[len(f.events)-1] != meshwants.EventWaitEnded {
t.Fatalf("asg at %s, waits %v, events %v", f.wantAt("asg"), f.waits(), f.events)
}
}
@@ -1,499 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The wait on an outside cause (design 47 G §706, ADR 0677): the assigner judges a node's health after a want moves, and a moved
// module's failing check on a machine whose open conditions name an outside cause is held as a wait (internal/outside has the
// rules). The conditions come only through the seat mesh-conditions: its verb `conditions`, read at each judging as a kept
// call, and its events, which only say when to judge again.
// conditionsVerb is the seat verb the keeper answers `conditions` by.
const conditionsVerb = "seat:" + mcseat.Seat + "." + mcseat.VerbConditions
// wantsPublisher says one of the seat mesh-wants' events, `wait-held` and `wait-ended` among them, under a message id the bus
// keeps one of.
func wantsPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatdef.EventSubject(meshwants.Seat, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", assignment.Module())
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
return err
}
}
// outsideActs keeps, while the process runs, what the judging of outside waits needs beside the kept waits: the conditions read
// for each machine (one answer is one judging), the machines a heard event says to judge again, the machines whose own network
// is held open, and each check's last signature.
type outsideActs struct {
mu sync.Mutex
call func(key string, body any) (json.RawMessage, error) // a kept call
wake func()
logf func(string, ...any)
// bound and settle are the settings outside-wait-minutes and outside-settle-seconds.
bound, settle time.Duration
// reading is a read in flight by machine, answered the answers it brought, not yet judged by.
reading map[string]bool
answered map[string][]outside.Cond
// dirty is a machine whose waits are judged again, cleared a wait's key whose cause's clearing was heard.
dirty map[string]bool
cleared map[string]bool
// sigs is each check's last signature, by node.module.
sigs map[string]string
// own is the machines whose own network is open; ownRead says the read at the start was answered.
own map[string]bool
ownRead bool
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
causes map[string][]string
// started says the first pass judged every wait again.
started bool
}
func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func(), logf func(string, ...any), bound, settle time.Duration) *outsideActs {
if bound <= 0 {
bound = outside.DefaultBound
}
if settle <= 0 {
settle = outside.DefaultSettle
}
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, own: map[string]bool{}, causes: map[string][]string{}}
}
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
// each read at the start from the file the manifest declares with the content ${config:<key>}, which the environment names
// (A-112, as the builder reads its own); zero for one whose file is missing or empty, which is its default, and an error
// naming the file for a content that is no positive whole number.
func outsideSettings(getenv func(string) string) (bound, settle time.Duration, err error) {
minutes, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_WAIT_FILE"))
if err != nil {
return 0, 0, err
}
seconds, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"))
if err != nil {
return 0, 0, err
}
return time.Duration(minutes) * time.Minute, time.Duration(seconds) * time.Second, nil
}
// settingFile reads one setting's file: 0 for no path, no file or an empty one.
func settingFile(path string) (int, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
}
return n, nil
}
// readStart reads the open conditions of every machine once at the start, for the machines whose own network is held.
func (o *outsideActs) readStart() {
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine"})
if err != nil {
o.logf("the open conditions are not read at the start (%v): no machine is held for its own network until its event is heard", err)
return
}
conds, err := outside.ParseConditions(answer)
if err != nil {
o.logf("%v", err)
return
}
o.mu.Lock()
for m := range outside.OwnNetwork(conds) {
o.own[m] = true
}
o.ownRead = true
o.mu.Unlock()
o.wake()
}()
}
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again and keeps the
// machines whose own network is open; nothing is kept of the condition.
func (o *outsideActs) Heard(subject string, body []byte) {
var e mcseat.ConditionEvent
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
o.logf("%s holds what is not a condition's event, and is dropped: %v", subject, err)
return
}
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
o.mu.Lock()
if p := strings.Split(e.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" {
o.own[p[1]] = !cleared
}
if cleared {
for _, node := range o.causes[e.Key] {
o.dirty[node] = true
o.cleared[node+"|"+e.Key] = true
}
} else if e.Subject != nil {
o.dirty[e.Subject.Machine] = true
for _, m := range e.Subject.Also {
o.dirty[m] = true
}
}
o.mu.Unlock()
o.wake()
}
// Held says whether machine's own network is open: no want moves to it.
func (o *outsideActs) Held(machine string) bool {
if o == nil {
return false
}
o.mu.Lock()
defer o.mu.Unlock()
return o.own[machine]
}
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
// which it starts when none is in flight.
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
o.mu.Lock()
defer o.mu.Unlock()
if c, ok := o.answered[machine]; ok {
delete(o.answered, machine)
return c, true
}
if o.reading[machine] {
return nil, false
}
o.reading[machine] = true
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine", "machine": machine})
var conds []outside.Cond
if err == nil {
conds, err = outside.ParseConditions(answer)
}
o.mu.Lock()
o.reading[machine] = false
if err != nil {
o.logf("the conditions of %s are not read (%v): the move waits on %s", machine, err, outside.OnUnread)
} else {
o.answered[machine] = conds
o.dirty[machine] = true // the kept call answered: the move is judged by it
}
o.mu.Unlock()
if err == nil {
o.wake()
}
}()
return nil, false
}
// outsideReview is the judging of module's check at fp on each of its nodes (§706): it returns the nodes as the rules read them,
// the have of a waiting move no failure (its health unknown) and a go-on's counted healthy, and keeps the waits, says their
// events, finding and ask. moved says the delivery moved owner to node.
func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []string, n assign.Nodes, keeperSelf bool, moved func(owner, node string) bool) assign.Nodes {
o := p.outside
if o == nil || fp == "" || p.kvs[buckets.Waits()] == nil {
return n
}
hex, err := catalogkey.KeyPart(fp)
if err != nil {
return n
}
now := p.clock()
kept := p.w.read(buckets.Waits())
override := map[string]string{}
// A wait of another fingerprint of this module is replaced by the newer want.
for key, raw := range kept {
if w, err := outside.Decode(raw); err == nil && w.Module == module && w.FP != fp {
p.outsideEnd(ctx, key, w, outside.HowReplaced, now)
}
}
for _, node := range nodes {
key := outside.Key(node, module, hex)
var prev *outside.Wait
if raw := kept[key]; raw != nil {
if w, err := outside.Decode(raw); err == nil {
prev = &w
}
}
h, ok := n.Haves[node][module]
sig := fmt.Sprintf("%s|%s|%s", h.Fp, h.Phase, h.Health)
o.mu.Lock()
first := !o.started
changed := o.sigs[node+"."+module] != sig
o.sigs[node+"."+module] = sig
dirty := o.dirty[node]
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
o.mu.Unlock()
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
if !failing && !healthy && prev == nil {
continue
}
if prev != nil && p.outsideAnswered(prev, node, module, fp) {
dirty = true
}
due := prev != nil && (prev.Release != "" || (prev.Settling && !now.Before(prev.SettleEnds)))
var d outside.Decision
if prev != nil && !(first || changed || dirty || due) {
// nothing to judge by: the wait stands as kept; the bound's ask is made when it has run
d = outside.Decision{Wait: prev, Waiting: prev.Standing() || prev.Settling, Own: prev.Decided == outside.HowFailed || prev.Decided == outside.HowOwnFailure,
Counted: prev.Decided == outside.HowGoOn}
if prev.Standing() && !now.Before(prev.ClockFrom.Add(o.bound)) {
d.Ask = true
}
} else {
f := outside.Facts{Now: now, Failing: failing, Healthy: healthy, Node: node, KeeperSelf: keeperSelf, Bound: o.bound, Settle: o.settle,
Moved: func(owner string) bool { return moved(owner, node) }, CauseCleared: cleared}
if failing && !keeperSelf {
f.Conds, f.Read = o.take(node)
}
d = outside.Judge(prev, node, module, fp, f)
o.mu.Lock()
delete(o.dirty, node)
if prev != nil {
delete(o.cleared, node+"|"+prev.Cause)
}
o.mu.Unlock()
}
switch {
case d.Waiting:
override[node] = nodeengineclient.HealthUnknown
case d.Counted:
override[node] = nodeengineclient.HealthOK
}
p.outsideApply(ctx, key, hex, node, module, fp, prev, d, now)
}
o.mu.Lock()
o.started = true
o.mu.Unlock()
if len(override) == 0 {
return n
}
out := n
out.Haves = map[string]map[string]nodeengine.HaveByNodeModuleValue{}
for node, byModule := range n.Haves {
out.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
for m, h := range byModule {
out.Haves[node][m] = h
}
}
for node, health := range override {
h := out.Haves[node][module]
h.Health = health
if health == nodeengineclient.HealthOK {
h.Phase = nodeengineclient.PhaseRunning
}
out.Haves[node][module] = h
}
return out
}
// outsideAnswered reads the operator's answer to the wait's ask, from the teller's record of it under the ask's key (answeredWith,
// §700's path as the assigner reads an ask's answer), and notes it on the wait as a release; it says whether it did.
func (p *passer) outsideAnswered(w *outside.Wait, node, module, fp string) bool {
if w.Ask == "" || w.Release != "" {
return false
}
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return false
}
for _, how := range []string{assign.OptionGoOn, assign.OptionFailed} {
if p.answeredWith(key, how) {
w.Release, w.By, w.Why = how, "the operator", "the answer to the ask "+w.Ask
return true
}
}
return false
}
// outsideApply keeps what a judging came to: the wait's record, its events, its finding and its ask.
func (p *passer) outsideApply(ctx context.Context, key, hex, node, module, fp string, prev *outside.Wait, d outside.Decision, now time.Time) {
o := p.outside
switch {
case d.End != "":
p.outsideEnd(ctx, key, *prev, d.End, now)
if d.Wait != nil {
p.keepWait(ctx, key, *d.Wait)
}
return
case d.Wait == nil:
if prev != nil {
p.outsideEnd(ctx, key, *prev, "", now)
}
return
}
w := *d.Wait
if d.Held {
body := p.outsideBody(w)
if err := p.wantsSay(meshwants.EventWaitHeld, body, key+"."+w.On+"."+w.ClockFrom.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the wait of %s on %s is not said, and is said at the next pass: %v", module, node, err)
return
}
p.outsideFinding(w, true)
}
if d.Ask && w.Ask == "" {
if id := p.outsideAsk(ctx, node, module, fp, w, now); id != "" {
w.Ask = id
}
}
if w.Standing() || w.Settling || w.Decided != "" {
o.mu.Lock()
if w.Cause != "" && w.Standing() {
if !containsString(o.causes[w.Cause], node) {
o.causes[w.Cause] = append(o.causes[w.Cause], node)
}
}
o.mu.Unlock()
if w.Settling {
p.wakeAt(w.SettleEnds, now)
} else if w.Standing() {
p.wakeAt(w.ClockFrom.Add(o.bound), now)
}
p.keepWait(ctx, key, w)
}
}
func containsString(l []string, s string) bool {
for _, x := range l {
if x == s {
return true
}
}
return false
}
// keepWait writes the wait's record.
func (p *passer) keepWait(ctx context.Context, key string, w outside.Wait) {
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return outside.Encode(w) })
}
func (p *passer) outsideBody(w outside.Wait) meshwants.Wait {
since, from := w.Since, w.ClockFrom
return meshwants.Wait{Step: meshwants.WaitStepOutside, Owner: assignment.Module(), Node: w.Node, Module: w.Module, Fingerprint: w.FP, On: w.On,
Since: since, ClockFrom: &from, Bound: w.Bound,
Words: fmt.Sprintf("%s on %s waits on %s", w.Module, w.Node, w.On)}
}
// outsideEnd ends a wait: its wait-ended with how (none for a settle, which was never said), its finding and ask taken away, and
// its record deleted unless the end decided it (the caller keeps that record).
func (p *passer) outsideEnd(ctx context.Context, key string, w outside.Wait, how string, now time.Time) {
if w.Standing() && how != "" {
body := p.outsideBody(w)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+"."+how+"."+now.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the end of the wait of %s on %s is not said, and is said at the next pass: %v", w.Module, w.Node, err)
return
}
}
p.outsideFinding(w, false)
if ask, err := asks.OutsideAskKey(w.Node, w.Module, w.FP); err == nil {
p.remove(ctx, buckets.Asks(), ask)
}
if p.outside != nil && w.Cause != "" {
p.outside.mu.Lock()
p.outside.causes[w.Cause] = removeString(p.outside.causes[w.Cause], w.Node)
p.outside.mu.Unlock()
}
p.remove(ctx, buckets.Waits(), key)
}
func removeString(l []string, s string) []string {
var out []string
for _, x := range l {
if x != s {
out = append(out, x)
}
}
return out
}
// outsideFinding writes the finding delivery-waits-outside, or clears it.
func (p *passer) outsideFinding(w outside.Wait, raised bool) {
if p.findings.Put == nil {
return
}
o := assign.WaitsOutsideFinding(w.Node, w.Module, w.Cause, w.On, p.clock())
if raised {
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", "delivery."+o.ID, err)
}
return
}
if p.findings.Delete != nil {
if err := p.findings.Delete(mcseatKey(o)); err != nil {
p.logf("the finding %s is not cleared: %v", "delivery."+o.ID, err)
}
}
}
func mcseatKey(o assign.Observed) string {
return views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
}
// outsideAsk asks the operator at the wait's bound, once, and returns the asking's id.
func (p *passer) outsideAsk(ctx context.Context, node, module, fp string, w outside.Wait, now time.Time) string {
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return ""
}
asking := assign.AskingSeconds(now)
a := assign.OutsideAsk(key, asks.OutsideAbout(key), node, module, fp, w.On, w.ClockFrom, asking, now)
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
if ask, err := assign.AskOf(a); err == nil {
return ask.ID
}
return ""
}
// wakeAt makes a pass when t comes: the settle's end and the bound are the waits' own times, the one wake (no polling).
func (p *passer) wakeAt(t, now time.Time) {
if p.outside == nil || p.outside.wake == nil {
return
}
time.AfterFunc(max(t.Sub(now), 0)+time.Millisecond, p.outside.wake)
}
@@ -1,333 +0,0 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The whole of the wait on an outside cause in the pass (design 47 G §706): a fixture keeper answering `conditions` and saying
// its events, a failing check of nats on ace, and the events, finding, ask and record the pass leaves.
type outsideRig struct {
t *testing.T
p *passer
now time.Time
conds []map[string]any
unread bool
events []string
bodies []meshwants.Wait
found map[string]bool
woke chan struct{}
health string
}
func newOutsideRig(t *testing.T) *outsideRig {
r := &outsideRig{t: t, now: time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC), found: map[string]bool{}, woke: make(chan struct{}, 16), health: nodeengineclient.HealthFailed}
r.p = newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, r.now)
r.p.now = func() time.Time { return r.now }
r.p.w.put(buckets.Asks(), "x", nil)
r.p.outside = newOutsideActs(func(key string, body any) (json.RawMessage, error) {
if r.unread {
return nil, context.DeadlineExceeded
}
return json.Marshal(map[string]any{"conditions": r.conds})
}, func() { r.woke <- struct{}{} }, func(string, ...any) {}, 30*time.Minute, 90*time.Second)
r.p.publishWants = func(event string, body any, id string) error {
r.events = append(r.events, event)
r.bodies = append(r.bodies, body.(meshwants.Wait))
return nil
}
r.p.findings.Put = func(f mcseat.Finding) error { r.found[f.Scope+"."+f.ID+"."+f.Kind] = true; return nil }
r.p.findings.Delete = func(key string) error { delete(r.found, key); return nil }
return r
}
func (r *outsideRig) cond(key string, also ...string) {
r.conds = append(r.conds, map[string]any{"key": key, "subject": map[string]any{"machine": strings.Split(key, ".")[1], "also": also}})
}
// review is one pass's judging of nats on ace at the fingerprint, with the read answered at once when one is started.
func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) assign.Nodes {
r.t.Helper()
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
healthy := r.health == nodeengineclient.HealthOK
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
if moved == nil {
moved = func(string, string) bool { return false }
}
out := r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
select {
case <-r.woke: // the read was answered: the next pass judges by it
out = r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
case <-time.After(50 * time.Millisecond):
}
return out
}
func (r *outsideRig) wait() outside.Wait {
r.t.Helper()
for _, v := range r.p.w.read(buckets.Waits()) {
w, err := outside.Decode(v)
if err != nil {
r.t.Fatal(err)
}
return w
}
return outside.Wait{}
}
func TestTheWaitOnAnOutsideCause(t *testing.T) {
healthOf := func(n assign.Nodes) string { return n.Haves["ace"]["nats"].Health }
t.Run("a cause listing the machine is a wait said at once, ended cleared when it clears healthy", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 1 || r.events[0] != meshwants.EventWaitHeld {
t.Fatalf("health %s events %v", healthOf(n), r.events)
}
b := r.bodies[0]
if b.Step != meshwants.WaitStepOutside || b.On != "unreachable:hub" || b.Bound != "30m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(r.now) || b.Node != "ace" || b.Module != "nats" {
t.Errorf("wait-held %+v", b)
}
if !r.found["delivery.ace.nats.delivery-waits-outside"] {
t.Errorf("findings %v", r.found)
}
// the cause clears and the module runs healthy
r.conds, r.health, r.now = nil, nodeengineclient.HealthOK, r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.hub.unreachable","event":"cleared","change":"cleared","at":"2026-11-11T10:01:00Z","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
if len(r.events) != 2 || r.events[1] != meshwants.EventWaitEnded || r.bodies[1].How != outside.HowCleared || len(r.found) != 0 || len(r.p.w.read(buckets.Waits())) != 0 {
t.Errorf("events %v bodies %+v findings %v", r.events, r.bodies, r.found)
}
})
t.Run("a cause not listing the machine settles, then the failure is the delivery's", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "g14")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 0 || !r.wait().Settling {
t.Fatalf("settle: health %s events %v wait %+v", healthOf(n), r.events, r.wait())
}
r.now = r.now.Add(91 * time.Second)
if n := r.review(false, nil); healthOf(n) != nodeengineclient.HealthFailed {
t.Errorf("past the settle the check still counts as failing: %s", healthOf(n))
}
})
t.Run("a cause turning into another keeps the wait and its clock; a rewritten naming a moved module fails at once", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
r.conds = nil
r.cond("machine.hub.network", "ace")
r.now = r.now.Add(5 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.network","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
last := len(r.bodies) - 1
if len(r.events) != 2 || r.events[last] != meshwants.EventWaitHeld || r.bodies[last].On != "network:hub" || !r.bodies[last].ClockFrom.Equal(*r.bodies[0].ClockFrom) {
t.Errorf("events %v bodies %+v", r.events, r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.ace.nats.rewritten")
n := r2.review(false, func(owner, node string) bool { return owner == "nats" })
if healthOf(n) == nodeengineclient.HealthFailed || !r2.wait().Settling {
t.Errorf("a rewritten of the moved module is its failure: %s %+v", healthOf(n), r2.wait())
}
})
t.Run("the keeper silent waits on conditions-unread, and its own module is judged with no read", func(t *testing.T) {
r := newOutsideRig(t)
r.unread = true
r.review(false, nil)
r.now = r.now.Add(90 * time.Second) // the settle has run with no answer
r.review(false, nil)
if len(r.bodies) != 1 || r.bodies[0].On != outside.OnUnread {
t.Fatalf("bodies %+v", r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.hub.unreachable", "ace")
if n := r2.review(true, nil); healthOf(n) != nodeengineclient.HealthFailed || len(r2.events) != 0 {
t.Errorf("the keeper's own failing check waits: %s %v", healthOf(n), r2.events)
}
})
t.Run("past the bound the operator is asked; Go on counts it healthy and Count as failed holds it, each said ended", func(t *testing.T) {
for how, want := range map[string]string{assign.OptionGoOn: nodeengineclient.HealthOK, assign.OptionFailed: nodeengineclient.HealthFailed} {
r := newOutsideRig(t)
r.cond("machine.hub.silent", "ace")
r.review(false, nil)
r.now = r.now.Add(31 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-changed", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
key, _ := asks.OutsideAskKey("ace", "nats", fpTest)
raw := r.p.w.value(buckets.Asks(), key)
if raw == nil {
t.Fatalf("%s: no ask at %s", how, key)
}
_, ask, err := assign.DecodeAsk(key, raw)
if err != nil || ask.About != "ace.nats."+strings.TrimPrefix(key[strings.LastIndex(key, ".")+1:], "") || len(ask.Options) != 2 {
t.Fatalf("ask %+v %v", ask, err)
}
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: how, Warrant: ask.ID + "@2026-11-11T10:32:00Z", Words: "the operator"})
r.p.w.put(buckets.Answers, key, v)
n := r.review(false, nil)
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != how || healthOf(n) != want {
t.Errorf("%s: events %v last %+v health %s", how, r.events, r.bodies[len(r.bodies)-1], healthOf(n))
}
if r.p.w.value(buckets.Asks(), key) != nil || len(r.found) != 0 {
t.Errorf("%s: the ask or the finding stands after the end", how)
}
// a check that goes on failing is not waited on again
r.now = r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
before := len(r.events)
r.review(false, nil)
if len(r.events) != before {
t.Errorf("%s: a decided check is waited on again", how)
}
}
})
t.Run("a newer want ends the wait replaced, and a restart judges it again", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}}
r.p.outsideReview(context.Background(), "nats", "src2:"+strings.Repeat("c", 64), []string{"ace"}, n, false, func(string, string) bool { return false })
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != outside.HowReplaced {
t.Errorf("events %v", r.events)
}
})
t.Run("its bounds are read at the start from the files the manifest declares (A-112)", func(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
env := map[string]string{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": write("wait", "45\n"), "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": write("settle", "7")}
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 45*time.Minute || settle != 7*time.Second {
t.Errorf("bound %v settle %v err %v, want 45m and 7s", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_WAIT_FILE"], env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = filepath.Join(dir, "missing"), write("empty", "")
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 0 || settle != 0 {
t.Errorf("bound %v settle %v err %v, want both defaults", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = write("bad", "soon")
if _, _, err := outsideSettings(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") {
t.Errorf("err %v, want one naming the file", err)
}
})
}
func TestAMachineWhoseOwnNetworkIsOpenGetsNoWant(t *testing.T) {
m := assign.Module{Name: "nats", FP: fpTest, Built: true, Nodes: []string{"ace"}, MachineHeld: func(node string) bool { return node == "ace" }}
ok, rule, why := assign.Holds(m, "ace", assign.Nodes{Online: map[string]bool{"ace": true}})
if ok || rule == 0 || !strings.Contains(why, "machine.ace.network") {
t.Errorf("holds = %v %d %q", ok, rule, why)
}
o := newOutsideActs(nil, func() {}, func(string, ...any) {}, 0, 0)
o.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if !o.Held("ace") || o.Held("g14") {
t.Error("the own network condition's event does not hold the machine")
}
o.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if o.Held("ace") {
t.Error("the machine is held after its condition cleared")
}
}
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
for k, v := range r.p.w.read(buckets.Waits()) {
s.st.put(waitsState, k, v)
}
got, err := s.doWaits(map[string]any{"module": "nats"})
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
t.Errorf("wait = %+v", w)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("an unknown how is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on"}, "shanks.node-tools"); err == nil || len(acts) != 0 {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on", "why": "the hub is being rebuilt"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if len(acts) == 0 || !strings.Contains(acts[0], "hand-act") {
t.Errorf("the hand act is not recorded first: %v", acts)
}
keys, _ := s.st.keys(waitsState)
raw, _, _, _ := s.st.get(waitsState, keys[0])
if w, _ := outside.Decode(raw); w.Release != "go-on" || !strings.Contains(w.By, "the hub is being rebuilt") {
t.Errorf("wait = %+v", w)
}
// the process takes the release at its next pass: ended go-on, counted healthy
r.p.w.put(buckets.Waits(), keys[0], raw)
n := r.review(false, nil)
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
}
// A-246: a dependent's delivery wait is listed beside them, and released by its module and step delivery, its hand act first.
dw, _ := json.Marshal(deliveryWaiting{Wait: meshwants.Wait{Step: meshwants.WaitStepDelivery, Module: "app", Part: "db", On: "db@1.1.0", Machines: []string{"g14"}}})
s.st.put(buckets.DeliveryWaitsState, "app", dw)
got, err = s.doWaits(map[string]any{"node": "g14"})
if d := got.(map[string]any)["delivery"].([]deliveryWaiting); err != nil || len(d) != 1 || d[0].Module != "app" {
t.Errorf("the delivery waits listed = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"module": "app", "step": "build", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of another step is taken")
}
if _, err := s.doRelease(map[string]any{"module": "web", "step": "delivery", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of a module waiting for no delivery is taken")
}
before := len(acts)
if _, err := s.doRelease(map[string]any{"module": "app", "step": "delivery", "why": "db is fine as it runs"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
raw, _, _, _ = s.st.get(buckets.DeliveryWaitsState, "app")
var released deliveryWaiting
if json.Unmarshal(raw, &released) != nil || !released.Released || !strings.Contains(released.By, "db is fine") || len(acts) != before+1 {
t.Errorf("the delivery wait after its release = %+v, hand acts %v", released, acts[before:])
}
// A-250: assigner_delivered_again records its hand act, writes the ask and answers asked; assigner_waits lists it.
if _, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0"}, "op"); err == nil {
t.Error("an ask with no why is taken")
}
ans, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0", "why": "the catalog missed it"}, "op")
if err != nil || ans.(map[string]any)["asked"] != "db" || len(acts) != before+2 {
t.Fatalf("assigner_delivered_again = %+v %v", ans, err)
}
got, _ = s.doWaits(map[string]any{})
if a := got.(map[string]any)["delivered-again"].([]againAsk); len(a) != 1 || a[0].Version != "1.1.0" || a[0].By != "op" {
t.Errorf("the asks listed = %+v", a)
}
}
@@ -1,300 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The tools of the waits on an outside cause (design 47 G §706, §276's row *delivery on an outside cause*): `assigner_waits` reads
// each, the settle included, and `assigner_release` with `how` takes one. The tool writes the release into the wait's record and the
// assigner's process ends the wait at its next pass, saying wait-ended with that how.
// toolWaits is the tool that reads the outside waits.
const toolWaits = "assigner_waits"
// waitsState is the assigner's state of waits, as the runtime names a module's own state.
const waitsState = buckets.WaitsState
// waitSaid is one wait as assigner_waits answers it: its record and where it stands.
type waitSaid struct {
Key string `json:"key"`
outside.Wait
// Settling's end is in the record; Ends is when the bound runs from clock-from, empty for a settle.
BoundEnds string `json:"bound-ends,omitempty"`
}
// actingSaid is one wait for an acting part's new version as assigner_waits answers it (actingwaits.go).
type actingSaid struct {
Key string `json:"key"`
acting.Wait
BoundEnds string `json:"bound-ends,omitempty"`
}
// doWaits is the tool assigner_waits.
func (s *putTools) doWaits(args map[string]any) (any, error) {
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
sort.Strings(keys)
out := []any{}
for _, key := range keys {
raw, _, found, err := s.st.get(waitsState, key)
if err != nil || !found {
continue
}
if acting.Is(key) || acting.IsOrder(key) {
if a, err := acting.Decode(raw); err == nil && (arg(args, "node") == "" || a.Node == arg(args, "node")) && (arg(args, "module") == "" || a.Module == arg(args, "module")) {
said := actingSaid{Key: key, Wait: a}
if d, err := time.ParseDuration(a.Bound); err == nil && !a.Released {
said.BoundEnds = a.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
out = append(out, said)
}
continue
}
w, err := outside.Decode(raw)
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
continue
}
said := waitSaid{Key: key, Wait: w}
if w.Standing() {
if d, err := time.ParseDuration(w.Bound); err == nil {
said.BoundEnds = w.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
}
out = append(out, said)
}
// And each dependent's wait for its dependency's delivery (design 47 G §376, A-246), from the state delivery-waits.
delivery := []deliveryWaiting{}
dkeys, err := s.st.keys(buckets.DeliveryWaitsState)
if err != nil {
return nil, fmt.Errorf("the delivery waits cannot be listed: %w", err)
}
sort.Strings(dkeys)
for _, key := range dkeys {
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil || !found {
continue
}
var w deliveryWaiting
if json.Unmarshal(raw, &w) != nil || (arg(args, "module") != "" && w.Module != arg(args, "module")) ||
(arg(args, "node") != "" && !slices.Contains(w.Machines, arg(args, "node"))) {
continue
}
delivery = append(delivery, w)
}
// And each ask of assigner_delivered_again not yet holding, with why (A-250).
again := []againAsk{}
akeys, err := s.st.keys(buckets.DeliveredAgainState)
if err != nil {
return nil, fmt.Errorf("the asks to say a version delivered again cannot be listed: %w", err)
}
sort.Strings(akeys)
for _, key := range akeys {
raw, _, found, err := s.st.get(buckets.DeliveredAgainState, key)
var a againAsk
if err != nil || !found || json.Unmarshal(raw, &a) != nil || (arg(args, "module") != "" && a.Module != arg(args, "module")) {
continue
}
again = append(again, a)
}
return map[string]any{"waits": out, "delivery": delivery, "delivered-again": again}, nil
}
// toolDeliveredAgain is the tool that asks version-delivered said again for one version (design 47 G §376, A-250).
const toolDeliveredAgain = "assigner_delivered_again"
// doDeliveredAgain is assigner_delivered_again: the hand act is recorded first, then the ask is written into the state
// delivered-again, whose watch the assigner's process hears at once; it answers that it is asked.
func (s *putTools) doDeliveredAgain(args map[string]any, by string) (any, error) {
module, version, why := arg(args, "module"), arg(args, "version"), arg(args, "why")
if module == "" || version == "" || why == "" {
return nil, errors.New("assigner_delivered_again names the module, its version and why: it is kept in the hand-act log")
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_delivered_again %s %s", module, version), Why: why,
Cause: "version-delivered", Verb: "mesh-wants.assigner_delivered_again"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err)
}
data, err := json.Marshal(againAsk{Module: module, Version: version, By: by, Why: why})
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveredAgainState, keys.EscapeName(module), data); err != nil {
return nil, fmt.Errorf("the ask of %s %s is not written: %w", module, version, err)
}
return map[string]any{"asked": module, "version": version, "hand-act": taken.Log,
"follow": "the assigner says version-delivered again once every present machine placing it runs it healthy; assigner_waits says why it does not yet"}, nil
}
// deliveredAgainTool is assigner_delivered_again as the runtime serves it.
func (s *putTools) deliveredAgainTool() stdio.Tool {
return stdio.Tool{Name: toolDeliveredAgain, Description: "Say version-delivered again for one version of a module, for the operator and the healers " +
"(design 47 G §376): written in the hand-act log first, then asked of the assigner, which says it once every present machine placing " +
"the module runs that version healthy; until then assigner_waits lists the ask with why it does not hold. Answers that it is asked.",
Input: schema(map[string]string{"module": "the module", "version": "its MAJOR.MINOR.PATCH", "why": "why, kept in the hand-act log"},
[]string{"module", "version", "why"}),
Run: func(a map[string]any) (any, error) { return s.doDeliveredAgain(a, stdio.CallerOf(a)) }}
}
// doReleaseDelivery is assigner_release with step delivery: the hand act is recorded first, then the release is written on the
// module's delivery wait, which the assigner's process ends released at its next pass, writing the module's want.
func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error) {
module, why := arg(args, "module"), arg(args, "why")
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
key := keys.EscapeName(module)
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil {
return nil, fmt.Errorf("the delivery wait of %s is not read: %w", module, err)
}
var w deliveryWaiting
if !found || json.Unmarshal(raw, &w) != nil || w.Released {
return nil, fmt.Errorf("%s waits for no delivery: assigner_waits lists the delivery waits", module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s step delivery (waits on %s)", module, w.On), Why: why,
Cause: "delivery-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Released, w.By = true, by+": "+why
data, err := json.Marshal(w)
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveryWaitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s's delivery wait is not written: %w", module, err)
}
return map[string]any{"released": module, "step": meshwants.WaitStepDelivery, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
// module, which the assigner's process takes at its next pass.
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
if how == acting.HowNow {
return s.doReleaseActing(node, module, why, by)
}
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
}
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
prefix := node + "." + module + "."
var key string
var w outside.Wait
for _, k := range keys {
if !strings.HasPrefix(k, prefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if c, err := outside.Decode(raw); err == nil && c.Decided == "" {
key, w = k, c
}
}
if key == "" {
return nil, fmt.Errorf("%s holds no wait of %s on an outside cause: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, how), Why: why,
Cause: "outside-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Release, w.By, w.Why = how, by+": "+why, why
data, err := outside.Encode(w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseActing is assigner_release with how `now`: the entry of module on node moves with its acting parts' running versions. The
// hand act is recorded first; the release is written on each standing wait of the entry, and the process says wait-ended at its next pass.
func (s *putTools) doReleaseActing(node, module, why, by string) (any, error) {
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
type one struct {
key string
w acting.Wait
}
var held []one
prefix, orderPrefix := acting.Prefix+node+"."+module+".", acting.OrderPrefix+node+"."+module+"."
for _, k := range keys {
if !strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, orderPrefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if w, err := acting.Decode(raw); err == nil && !w.Released {
held = append(held, one{k, w})
}
}
if len(held) == 0 {
return nil, fmt.Errorf("%s holds no wait of %s for an acting part: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, acting.HowNow), Why: why,
Cause: "acting-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
released := []string{}
for _, h := range held {
h.w.Released, h.w.By, h.w.Why = true, by+": "+why, why
data, err := acting.Encode(h.w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, h.key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
released = append(released, h.key)
}
return map[string]any{"released": released, "how": acting.HowNow, "by": by, "hand-act": taken.Log}, nil
}
// waitsTool is assigner_waits as the runtime serves it.
func (s *putTools) waitsTool() stdio.Tool {
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: an entry waiting for an acting part's new version in composing or placing (§266, §268), with its part, version, step, clock-from and bound; and a moved module's failing health check on a " +
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
"machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil), Run: s.doWaits}
}
@@ -1,6 +0,0 @@
package main
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,539 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat
// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its
// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state`
// section `pins` (4 MiB), one key `trunk.<n>` per assignment graph revision whose value is the list of the records pinning
// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys:
// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision
// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names
// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up
// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being
// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state.
const (
pinsState = "pins"
pinsTool = "assigner_pins"
unpinTool = "assigner_unpin"
pinsMadeKey = "made"
pinsStatedKey = "pins-stated"
pinsCapBytes = 4 << 20
pinMissingKind = "pin-missing"
)
// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held
// when a record pinned it.
type PinsValue struct {
Records []string `json:"records"`
Missing bool `json:"missing,omitempty"`
}
// pinKey is the key of an assignment graph revision, its object's own name (A-169).
func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) }
// pinRevision is the revision a key of the state names, and whether it names one.
func pinRevision(key string) (int64, bool) {
n, ok := strings.CutPrefix(key, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// pinObjects is the object bucket of the revisions as the pins remove from it.
type pinObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// pinKeeper keeps the pins and removes what nothing pins.
type pinKeeper struct {
ctx context.Context
st endStore
objects pinObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
findings assign.Findings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// caught is whether the last event applied left none of the consumer's messages pending.
caught bool
}
// edit applies change to the value at key by compare-and-set, again while another write comes between.
func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error {
for {
raw, rev, err := k.st.get(key)
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %s do not read: %w", key, err)
}
}
next, write := change(v, raw != nil)
if !write {
return nil
}
value, _ := json.Marshal(next)
if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// held says the revision's object is kept.
func (k *pinKeeper) held(revision int64) (bool, error) {
_, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision))
if errors.Is(err, assigngraph.ErrNoObject) {
return false, nil
}
return err == nil, err
}
// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding
// pin-missing naming the record.
func (k *pinKeeper) pin(p mdseat.Pins) error {
for _, r := range p.Assignment {
held, err := k.held(r)
if err != nil {
return err
}
err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) {
if slices.Contains(v.Records, p.Record) && (held || v.Missing) {
return v, false
}
if !slices.Contains(v.Records, p.Record) {
v.Records = append(v.Records, p.Record)
slices.Sort(v.Records)
}
v.Missing = v.Missing || !held
return v, true
})
if err != nil {
return err
}
if !held {
k.missing(r, p.Record)
}
}
return nil
}
// free removes record from each assignment graph revision of p; a freed applied twice changes nothing.
func (k *pinKeeper) free(p mdseat.Pins) error {
for _, r := range p.Assignment {
err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) {
i := slices.Index(v.Records, p.Record)
if !found || i < 0 {
return v, false
}
v.Records = slices.Delete(v.Records, i, i+1)
return v, true
})
if err != nil {
return err
}
}
return nil
}
// missing writes the finding pin-missing, scope the assigner, id the revision.
func (k *pinKeeper) missing(revision int64, record string) {
if k.findings.Put == nil {
k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record)
return
}
o := assign.Observed{Scope: assignment.Module(), ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind,
Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision),
Headline: "A delivery record pins a revision the assigner no longer holds",
Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool),
Source: assignment.Module()}
if err := k.findings.Put(o.Finding(k.now())); err != nil {
k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err)
}
}
// made writes the key made once, create-only; one there already stays.
func (k *pinKeeper) made() error {
raw, _, err := k.st.get(pinsMadeKey)
if err != nil || raw != nil {
return err
}
value, _ := json.Marshal(k.now().UTC())
if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) {
return err
}
return nil
}
// timeAt is the time a key holds; false when it holds none.
func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) {
raw, _, err := k.st.get(key)
if err != nil || raw == nil {
return time.Time{}, false, err
}
var t time.Time
if err := json.Unmarshal(raw, &t); err != nil {
return t, false, fmt.Errorf("the key %s does not read: %w", key, err)
}
return t, true, nil
}
// stated keeps the at of a pins-stated applied.
func (k *pinKeeper) stated(at time.Time) error {
for {
raw, rev, err := k.st.get(pinsStatedKey)
if err != nil {
return err
}
var was time.Time
if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) {
return nil
}
value, _ := json.Marshal(at.UTC())
if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// statedSinceMade says a pins-stated said after the state was made was applied.
func (k *pinKeeper) statedSinceMade() (bool, error) {
made, ok, err := k.timeAt(pinsMadeKey)
if err != nil || !ok {
return false, err
}
at, ok, err := k.timeAt(pinsStatedKey)
if err != nil || !ok {
return false, err
}
return at.After(made), nil
}
// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied.
func (k *pinKeeper) hear(m jetstream.Msg) {
err := k.apply(m.Subject(), m.Data())
if err != nil {
k.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
caught := false
if meta, merr := m.Metadata(); merr == nil {
caught = meta.NumPending == 0
}
k.mu.Lock()
was := k.caught
k.caught = caught
k.mu.Unlock()
if caught && (!was || m.Subject() != mdseat.SubjectPinned) {
k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up
}
}
// apply takes one event's body.
func (k *pinKeeper) apply(subject string, data []byte) error {
switch subject {
case mdseat.SubjectPinned, mdseat.SubjectFreed:
var p mdseat.Pins
if err := json.Unmarshal(data, &p); err != nil || p.Record == "" {
k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err)
return nil
}
if subject == mdseat.SubjectPinned {
return k.pin(p)
}
return k.free(p)
case mdseat.SubjectPinsStated:
var s mdseat.PinsStatedEvent
if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() {
k.logf("%s holds no at, and is not taken: %v", subject, err)
return nil
}
return k.stated(s.At)
}
return nil // heads-replaced: a candidate's revision stops being current, judged after
}
// judge removes every revision neither pinned nor current, when the rule allows a removal at all.
func (k *pinKeeper) judge() {
k.mu.Lock()
caught := k.caught
k.mu.Unlock()
if !caught {
return
}
ok, err := k.statedSinceMade()
if err != nil {
k.logf("the pins are not judged: %v", err)
return
}
if !ok {
return // no pins-stated applied since the state was made: a pin said before may not be heard yet
}
if err := k.remove(); err != nil {
k.logf("the revisions nothing pins are not removed: %v", err)
}
}
// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted.
func (k *pinKeeper) remove() error {
names, err := k.objects.Names(k.ctx)
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := pinRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
raw, _, err := k.st.get(pinKey(r))
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %d do not read: %w", r, err)
}
}
if r == current || len(v.Records) > 0 {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
if raw != nil {
if err := k.st.drop(pinKey(r)); err != nil {
return fmt.Errorf("the pins of %d: %w", r, err)
}
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions no record pins and not current are removed", removed)
return nil
}
// hearPins registers the seat mesh-delivery's four events on the assigner's hub.
func hearPins(hub *process.EventHub, k *pinKeeper) error {
for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} {
if err := hub.Handle(s, k.hear); err != nil {
return fmt.Errorf("%s: %w", s, err)
}
}
return nil
}
// pinsTools are assigner_pins and assigner_unpin over the runtime's states.
func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " +
"the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " +
"was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.",
Run: func(map[string]any) (any, error) { return readPins(st) }},
{Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " +
"hand-act log first. The removal rule then applies.",
Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id",
"why": "why it is removed by hand"}, []string{"revision", "record", "why"}),
Run: func(a map[string]any) (any, error) {
record, _ := a["record"].(string)
why, _ := a["why"].(string)
return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a))
}}}
}
// readPins is what assigner_pins answers.
func readPins(st store) (any, error) {
ks, err := st.keys(pinsState)
if err != nil {
return nil, err
}
sort.Strings(ks)
revisions := map[string]PinsValue{}
out := map[string]any{"revisions": revisions}
var newest int64
bytes := 0
for _, key := range ks {
raw, _, found, err := st.get(pinsState, key)
if err != nil || !found {
continue
}
bytes += len(key) + len(raw)
if r, ok := pinRevision(key); ok {
var v PinsValue
if json.Unmarshal(raw, &v) == nil {
revisions[key] = v
}
newest = max(newest, r)
continue
}
out[key] = json.RawMessage(raw)
}
out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes}
if newest > 0 {
out["newest-pinned"] = newest
}
return out, nil
}
// unpin removes one record from one revision's list, the hand act recorded first.
func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) {
r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64)
switch {
case err != nil || r <= 0:
return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision)
case record == "" || why == "":
return nil, errors.New("name the record and say why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("an unpin names who asks it, and the runtime proved no caller")
}
key := pinKey(r)
raw, _, found, err := st.get(pinsState, key)
if err != nil {
return nil, err
}
var v PinsValue
if found {
if err := json.Unmarshal(raw, &v); err != nil {
return nil, err
}
}
i := slices.Index(v.Records, record)
if i < 0 {
return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool)
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record,
Why: why + " (" + by + ")", Cause: "unpin"})
if err != nil {
return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err)
}
v.Records = slices.Delete(v.Records, i, i+1)
value, _ := json.Marshal(v)
if err := st.put(pinsState, key, value); err != nil {
return nil, err
}
return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil
}
// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool
// process: the assigner's own state, heard as it changes, never on a timer.
func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) {
w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly())
if err != nil {
k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err)
return
}
go func() {
defer func() { _ = w.Stop() }()
for e := range w.Updates() {
if e == nil {
continue
}
if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut {
k.judge()
}
}
}()
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
func pinsBucket() string { return assignment.Module() + "_" + pinsState }
@@ -1,119 +0,0 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until
// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a
// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing,
// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays.
func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) {
objects := memObjects{}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
return d
}
shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
// A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept.
cand := entry("d", "fp-d2")
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
var found []mcseat.Finding
clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock },
logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" },
findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}}
if err := k.made(); err != nil {
t.Fatal(err)
}
say := func(subject string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if err := k.apply(subject, raw); err != nil {
t.Fatal(err)
}
k.judge()
}
pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} }
has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok }
k.caught = true
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r2", 1))
say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed
if !has(2) {
t.Fatal("revision 2 was removed before a pins-stated said after the state was made")
}
k.caught = false
say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)})
if !has(2) {
t.Fatal("revision 2 was removed while the consumer was behind")
}
k.caught = true
say(mdseat.SubjectFreed, pins("r1", 1))
if has(2) {
t.Error("revision 2, neither pinned nor current, is kept")
}
if !has(1) || !has(3) {
t.Error("revision 1 (pinned by r2) or 3 (current) was removed")
}
raw, _, _ := k.st.get(pinKey(1))
if string(raw) != `{"records":["r2"]}` {
t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw)
}
say(mdseat.SubjectFreed, pins("r2", 1))
if has(1) {
t.Error("revision 1 is kept once both records freed it")
}
if _, ok := objects[assigngraph.EntryObject(old)]; ok {
t.Error("the entry no kept revision lists and nothing names is kept")
}
if _, ok := objects[assigngraph.EntryObject(running)]; !ok {
t.Error("the entry of a running build is removed")
}
if _, ok := objects[assigngraph.EntryObject(shared)]; !ok {
t.Error("the entry the current revision lists is removed")
}
if _, ok := objects[assigngraph.EntryObject(cand)]; !ok {
t.Error("the entry a run's kept candidate lists is removed")
}
say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{})
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok {
t.Error("the gate's heads-replaced dropped a run's candidate")
}
say(mdseat.SubjectPinned, pins("r3", 2))
raw, _, _ = k.st.get(pinKey(2))
if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" {
t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found)
}
}
@@ -1,487 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as
// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and
// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written
// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing.
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
const (
writtenState = "written"
writtenTool = "assigner_written"
)
// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes.
const (
pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>"
)
// End is one value of the state written: the newest end of a module's pipeline on a watched branch.
type End struct {
Source int64 `json:"source"`
EndCount int64 `json:"end-count"`
Fp string `json:"fp"`
Commit string `json:"commit,omitempty"`
// Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed).
Passed bool `json:"passed"`
// Written: the wants were written from this end.
Written bool `json:"written,omitempty"`
Why string `json:"why,omitempty"`
}
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
func endKey(module, branch string) string {
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again.
func (e End) newer(source, count int64) bool {
return source > e.Source || (source == e.Source && count > e.EndCount)
}
// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed
// optional step holds nothing).
func requiredPassed(p mpr.PipelineEnded) bool {
for _, s := range p.Steps {
if s.Status != mpr.StepResultStatusRequired {
continue
}
switch s.Outcome {
case mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeCouldNotRun, mpr.StepResultOutcomeDeclined, mpr.StepResultOutcomeNotAsked,
mpr.StepResultOutcomeCancelled, mpr.StepResultOutcomeSuperseded, mpr.StepResultOutcomeDidNotBuild,
mpr.StepResultOutcomeRunning, mpr.StepResultOutcomeWaiting, mpr.StepResultOutcomeWaitingForABuildNode:
return false
}
}
return p.Outcome == mpr.PipelineEndedOutcomePassed
}
// endStore is the kept state written: the bus bucket, or a test's.
type endStore interface {
// get is a key's value and revision; none for a key not there.
get(key string) (value []byte, revision uint64, err error)
// swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not.
swap(key string, value []byte, revision uint64) error
// keys are the keys that stand, and drop deletes one.
keys() ([]string, error)
drop(key string) error
}
var errSwapped = errors.New("the key changed")
// kvEnds is endStore over a bus bucket.
type kvEnds struct {
ctx context.Context
kv jetstream.KeyValue
}
func (k kvEnds) get(key string) ([]byte, uint64, error) {
e, err := k.kv.Get(k.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (k kvEnds) swap(key string, value []byte, revision uint64) error {
var err error
if revision == 0 {
_, err = k.kv.Create(k.ctx, key, value)
} else {
_, err = k.kv.Update(k.ctx, key, value, revision)
}
if errors.Is(err, jetstream.ErrKeyExists) || (err != nil && isWrongSequence(err)) {
return errSwapped
}
return err
}
func (k kvEnds) keys() ([]string, error) {
ks, err := k.kv.Keys(k.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return ks, err
}
func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) }
func isWrongSequence(err error) bool {
var ae *jetstream.APIError
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
}
// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations
// (design 47 G §95, §353).
type pipelineEnds struct {
st endStore
wake func()
logf func(string, ...any)
// on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353).
on atomic.Bool
mu sync.Mutex
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
// 47 G §14).
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
trunk func() string
// runner is the module holding the seat mesh-pipeline-runner, as the last pass read it from the acting edges (A-334); nil
// until a pass read the graph.
runner atomic.Pointer[string]
// held are the group members' pipeline-ended heard before a pass read the graph, by their body: unacknowledged, so the
// bus keeps them, and taken once the runner is read (A-334), never dropped for the runner not being known yet.
held map[string]jetstream.Msg
}
// setRunner keeps the module holding the seat mesh-pipeline-runner, read by the pass from the acting edges (A-334); the
// first one takes the group members' ends held until the graph was read.
func (e *pipelineEnds) setRunner(module string) {
e.mu.Lock()
first := e.runner.Swap(&module) == nil
held := e.held
e.held = nil
e.mu.Unlock()
if !first {
return
}
for _, m := range held {
e.hearPipeline(m)
}
}
// holdUntilRead keeps m, a group member's pipeline-ended, until a pass reads the runner from the graph; false when the
// runner is read already, so m is taken now.
func (e *pipelineEnds) holdUntilRead(m jetstream.Msg) bool {
e.mu.Lock()
defer e.mu.Unlock()
if e.runner.Load() != nil {
return false
}
if e.held == nil {
e.held = map[string]jetstream.Msg{}
}
e.held[string(m.Data())] = m
e.logf("%s is a group member's end heard before the assignment graph is read: held until a pass reads which module is the runner (A-334)", m.Subject())
return true
}
// isRunner says whether module holds the seat mesh-pipeline-runner, as the last pass read it.
func (e *pipelineEnds) isRunner(module string) bool {
r := e.runner.Load()
return r != nil && *r != "" && *r == module
}
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
}
// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set.
func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error {
key := endKey(p.Module, p.Branch)
for {
raw, rev, err := e.st.get(key)
if err != nil {
return err
}
if raw != nil {
var kept End
if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) {
return nil // an older end, or this one again
}
}
end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fingerprint, Commit: p.Commit, Passed: passed, Why: p.Why}
value, err := json.Marshal(end)
if err != nil {
return err
}
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
return err
}
e.wake()
return nil
}
}
// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp.
func (e *pipelineEnds) markWritten(module, branch, fp string) {
key := endKey(module, branch)
for {
raw, rev, err := e.st.get(key)
var kept End
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written {
return
}
kept.Written = true
value, _ := json.Marshal(kept)
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
e.logf("the end of %s on %s is not marked written: %v", module, branch, err)
}
return
}
}
// passedAt says whether the newest end kept for module on branch passed at fp.
func (e *pipelineEnds) passedAt(module, branch, fp string) bool {
raw, _, err := e.st.get(endKey(module, branch))
var kept End
return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp
}
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
func (e *pipelineEnds) branchOf(module, placed string) string {
if placed != "" {
return placed
}
e.mu.Lock()
defer e.mu.Unlock()
if b := e.firsts[module]; b != "" {
return b
}
if e.trunk != nil {
return e.trunk()
}
return ""
}
// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, but the runner's: in a group
// take-in that changes the runner it is the acting part the other members wait on, so it goes first, wanted on its own end
// once that passed, ahead of the group (A-334, as the node-engine does, §268). A run on request delivers nothing (design 47 G
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)
case p.Run == "" && p.Group != "" && e.holdUntilRead(m):
return
case p.Run != "" || (p.Group != "" && !e.isRunner(p.Module)):
default:
if err := e.take(p, requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-<id>`) delivers
// nothing: it drops the run's candidate assignment graph (A-235).
func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
g, err := mprc.GroupEnded(m.Data())
if err != nil {
e.logf("%s is not taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
if strings.HasPrefix(g.Group, runGroupPrefix) {
if e.runEnded != nil {
if err := e.runEnded(g.Group); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearEnds registers the two events on the assigner's hub.
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
}
if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil {
return fmt.Errorf("%s: %w", groupEndedSubject, err)
}
return nil
}
// writtenTools is the tool assigner_written: every key of the state written, with its end.
func writtenTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it.",
Run: func(map[string]any) (any, error) {
ks, err := st.keys(writtenState)
if err != nil {
return nil, err
}
sort.Strings(ks)
out := map[string]End{}
for _, k := range ks {
raw, _, ok, err := st.get(writtenState, k)
if err != nil || !ok {
continue
}
var e End
if json.Unmarshal(raw, &e) == nil {
out[k] = e
}
}
return out, nil
}}}
}
// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one.
func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string {
if !e.following() {
return nil
}
seen := map[string]bool{}
var out []string
for _, b := range placed[module] {
b = e.branchOf(module, b)
if !seen[b] {
seen[b] = true
out = append(out, b)
}
}
if len(out) == 0 {
out = append(out, e.branchOf(module, ""))
}
sort.Strings(out)
return out
}
// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already
// (a node placed later, a module taken away).
func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool {
if !e.following() {
return true
}
for _, w := range wanted {
if w.Fp == fp {
return true
}
}
for _, b := range branches {
if !e.passedAt(module, b, fp) {
return false
}
}
return true
}
// wrote marks the end the wants of module were written from.
func (e *pipelineEnds) wrote(module, fp string, branches []string) {
if !e.following() {
return
}
for _, b := range branches {
e.markWritten(module, b, fp)
}
}
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first
// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone
// is a registration removed: all its ends go.
func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
e.mu.Lock()
if gone || len(reg.Branches) == 0 {
delete(e.firsts, reg.Module)
} else {
e.firsts[reg.Module] = reg.Branches[0]
}
e.mu.Unlock()
kept, err := e.st.keys()
if err != nil {
e.logf("the ends kept cannot be listed: %v", err)
}
prefix := keys.EscapeName(reg.Module) + "."
for _, k := range kept {
rest, ok := strings.CutPrefix(k, prefix)
if !ok {
continue
}
branch, err := keys.UnescapeName(rest)
if err == nil && !gone && slices.Contains(reg.Branches, branch) {
continue
}
if err := e.st.drop(k); err != nil {
e.logf("the end %s is not deleted: %v", k, err)
}
}
if e.onBranches != nil && !gone {
e.onBranches(reg.Branches)
}
if !e.on.Swap(true) {
e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module)
}
e.wake()
}
// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb
// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own
// goroutine, the wants keep the trunk-only form until a registration is heard.
func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error {
regs := mcc.NewRegistrations(call)
regs.Follower.Logf = e.logf
regs.Follower.Changed = func(module string, value json.RawMessage, gone bool, _ time.Time) {
reg := mpcat.Registration{Module: module}
if !gone {
_ = json.Unmarshal(value, &reg)
}
e.registered(reg, gone)
}
if err := hub.Handle(mpcat.SubjectRegistrationChanged, func(m jetstream.Msg) {
if err := regs.Hear(ctx, m.Data()); err != nil {
e.logf("%s: %v", m.Subject(), err)
}
_ = m.Ack()
}); err != nil {
return err
}
go func() {
if err := regs.Start(ctx); err != nil && ctx.Err() == nil {
e.logf("the first view of the registrations is not answered: %v", err)
}
}()
return nil
}
@@ -1,401 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
mws "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// memEnds is endStore over a map, with revisions.
type memEnds struct {
vals map[string][]byte
revs map[string]uint64
}
func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil }
func (m *memEnds) keys() (out []string, _ error) {
for k := range m.vals {
out = append(out, k)
}
return out, nil
}
func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil }
func (m *memEnds) swap(key string, v []byte, rev uint64) error {
if m.revs[key] != rev {
return errSwapped
}
m.vals[key], m.revs[key] = v, rev+1
return nil
}
// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back.
type heardMsg struct {
jetstream.Msg
data []byte
subject string
acked, naked bool
}
func (m *heardMsg) Data() []byte { return m.data }
func (m *heardMsg) Subject() string { return m.subject }
func (m *heardMsg) Ack() error { m.acked = true; return nil }
func (m *heardMsg) Nak() error { m.naked = true; return nil }
func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded {
return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps,
Pipeline: branch + "." + module, Commit: "c"}
}
func step(status, outcome string) mpr.StepResult {
return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome}
}
// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch.
func TestAWantFollowsAPassedPipeline(t *testing.T) {
const fp1, fp2 = "fp1", "fp2"
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional
for _, c := range []struct {
name string
ends []mpr.PipelineEnded
group *mpr.GroupEnded
module string
branch string
fp string
may bool
}{
{"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false},
{"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true},
{"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false},
{"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false},
{"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true},
{"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true},
{"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false},
{"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true},
{"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false},
} {
woken := 0
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
e.on.Store(true)
for _, p := range c.ends {
if p.Group != "" {
continue // heard and ignored, as hearPipeline does
}
if err := e.take(p, requiredPassed(p)); err != nil {
t.Fatal(err)
}
}
if g := c.group; g != nil {
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
t.Fatal(err)
}
}
}
if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may {
t.Errorf("%s: may want = %v, want %v", c.name, got, c.may)
}
}
// A run on request's group-ended (group run-<id>, A-235) writes no want and drops the run's candidate; a drop that fails is
// heard again.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
var dropped []string
fail := false
e.runEnded = func(group string) error {
dropped = append(dropped, group)
if fail {
return fmt.Errorf("the bucket is not open")
}
return nil
}
raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed,
Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}})
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"}
e.hearGroup(m)
if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked {
t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked",
e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked)
}
fail = true
m = &heardMsg{data: raw, subject: m.subject}
e.hearGroup(m)
if !m.naked {
t.Error("a run's candidate not dropped is not heard again")
}
}
// A-334 (§29, §65, §268): in a group take-in that changes the runner, the runner's member (the holder of the seat
// mesh-pipeline-runner, read from the acting edges) is wanted on its own pipeline-ended once it passed, ahead of the group;
// the other members wait for group-ended.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
heard := func(p mpr.PipelineEnded) *heardMsg {
p.Group = "g"
raw, _ := json.Marshal(p)
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.x"}
e.hearPipeline(m)
return m
}
hear := func(p mpr.PipelineEnded) {
if m := heard(p); !m.acked {
t.Errorf("the pipeline-ended of %s is not acknowledged", p.Module)
}
}
// The runner's end heard before a pass read the graph is held, unacknowledged, and taken once the runner is read.
early := heard(pipe("runner", "main", 5, 1, "fp-runner", pass, step(req, "passed")))
if early.acked || early.naked || e.mayWant("runner", "fp-runner", []string{"main"}, nil) {
t.Fatalf("the runner's end before the graph is read: acked %v, naked %v, wanted %v; want it held", early.acked, early.naked,
e.mayWant("runner", "fp-runner", []string{"main"}, nil))
}
e.setRunner(assigngraph.Runner([]mws.AssignmentEntry{{Node: "ace", Module: "web", Edges: []mws.Edge{
{Kind: assigngraph.EdgeActing, Node: "ace", Module: "runner", Seat: mpr.Seat, Step: "pipeline"}}}}))
if !early.acked {
t.Error("the runner's end held until the graph was read is not taken once it is")
}
hear(pipe("web", "main", 5, 1, fp1, pass, step(req, "passed")))
if !e.mayWant("runner", "fp-runner", []string{"main"}, nil) || e.mayWant("web", fp1, []string{"main"}, nil) {
t.Errorf("a group changing the runner: may want the runner %v, web %v; want the runner ahead of the group and web not yet",
e.mayWant("runner", "fp-runner", []string{"main"}, nil), e.mayWant("web", fp1, []string{"main"}, nil))
}
hear(pipe("runner", "main", 6, 1, "fp-runner-2", fail, step(req, "failed")))
if e.mayWant("runner", "fp-runner-2", []string{"main"}, nil) {
t.Error("the runner's member that failed is wanted")
}
}
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) {
t.Error("a want naming the fingerprint was held")
}
// markWritten keeps the end and says it was written from; a newer end replaces it unwritten.
st := newMemEnds()
e = newPipelineEnds(st, func() {}, t.Logf)
e.on.Store(true)
_ = e.take(pipe("web", "main", 5, 1, fp1, pass), true)
e.wrote("web", fp1, []string{"main"})
var end End
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if !end.Written || end.Source != 5 {
t.Errorf("end = %+v, want written at source 5", end)
}
_ = e.take(pipe("web", "main", 6, 1, fp2, pass), true)
end = End{}
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if end.Written || end.Source != 6 {
t.Errorf("a newer end = %+v, want unwritten at source 6", end)
}
}
// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at
// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before.
func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
for _, c := range []struct {
name string
follow bool
passed bool
want bool
}{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
mdigest := strings.Repeat("e", 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", fp, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
digest := "sha256:" + strings.Repeat("c", 64)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
composes(t, p.w, "g14", digest)
placeOn(t, p.w, "g14", "web")
on := true
p.w.heardPresence("g14", !on, time.Now())
if c.follow {
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.on.Store(true)
if c.passed {
_ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true)
}
}
p.wants(context.Background())
key, _ := mwc.WantKey("g14", "web")
if _, got := wantsKV.values[key]; got != c.want {
t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want)
}
if c.follow && c.passed {
raw, _, _ := p.ends.st.get(endKey("web", trunkBranch))
var end End
_ = json.Unmarshal(raw, &end)
if !end.Written {
t.Errorf("%s: the end was not marked written", c.name)
}
}
}
}
// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and
// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default.
func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) {
st := newMemEnds()
woken := 0
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
e.trunk = func() string { return trunkBranch }
_ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false)
_ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true)
if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) {
t.Fatal("the follower held a want before any registration")
}
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
if !e.following() || woken == 0 {
t.Error("the first registration did not switch the wants on")
}
if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) {
t.Error("after the switch the wants did not follow the ends")
}
if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", ""))
}
e.registered(mpcat.Registration{Module: "web"}, true)
if len(st.vals) != 0 {
t.Errorf("a registration gone left %d ends", len(st.vals))
}
}
// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a
// change on `beta` moves only the want of the node whose placement chose `beta`.
func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
digest := "sha256:" + strings.Repeat("c", 64)
on := true
for _, node := range []string{"g14", "ace"} {
composes(t, p.w, node, digest)
p.w.heardPresence(node, !on, time.Now())
}
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
}
// publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline.
publish := func(branch, f string, source int64) {
mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source)
if branch == "beta" {
mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source)
}
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, branch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", f, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", f)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
_ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true)
}
run := func(node, f string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, node+".web", have)
}
fpWant := func(node string) string {
key, _ := mwc.WantKey(node, "web")
w, err := mwc.DecodeWant(wantsKV.values[key])
if err != nil {
t.Fatalf("the want of web on %s: %v", node, err)
}
return w.Fp
}
publish("main", fp("a"), 5)
publish("beta", fp("b"), 5)
run("g14", fp("a"))
run("ace", fp("b"))
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
}
publish("beta", fp("d"), 6)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
}
}
// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node
// the rules read, and a registration's branches are each followed.
func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
w := newWorld()
woken := 0
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
entry := func(fp string) json.RawMessage {
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
return raw
}
h.changed("main", "web", entry("src2:1"), false)
h.changed("beta", "web", entry("src2:2"), false)
if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 {
t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken)
}
v := w.trunkOf("beta")["web"]
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
}
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
t.Errorf("graph node %+v", n)
}
h.changed("beta", "web", nil, true)
if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" {
t.Error("a module taken out of beta's half leaves main's")
}
}
@@ -1,95 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// placementBy is who a put the placement's build makes names as its asker: the placement, whose proposal the operator
// approved, is the warrant (design 47 *Going back is one act*; ADR 0666 §24).
const placementBy = "the placement"
// placementPuts makes each placed module's `build` a held put of that build, and releases a put the placement made once its
// placement names that build no more (design 47 line 912: "a return to an earlier build is one value in the placement, the
// module's `build` in its placement key, set by a placement proposal"; ADR 0666 §24; B-K1). The put then goes its own steps
// (the builder's build, the composition, an ask where the module's policy needs one, the want, holding) and is read with
// `assigner_puts` and released with `assigner_release` like any other; a placement's build the builder never built is refused
// by its build step, said in the put's words. A put asked by a person stands until the placement names another build.
func (p *passer) placementPuts(ctx context.Context) {
kv := p.kvs[buckets.Puts()]
if kv == nil {
return
}
now := p.clock().UTC()
named := map[string]string{} // put key -> the build its placement names
for node, a := range p.w.assignments() {
for _, m := range a.Modules {
if m.Build == "" {
continue
}
if key, err := puts.Key(node, m.Module); err == nil {
named[key] = m.Build
}
}
}
held := p.w.read(buckets.Puts())
keys := make([]string, 0, len(named)+len(held))
for k := range named {
keys = append(keys, k)
}
for k := range held {
if _, there := named[k]; !there {
keys = append(keys, k)
}
}
sort.Strings(keys)
for _, key := range keys {
build, names := named[key]
var before *puts.Put
if raw, ok := held[key]; ok {
if got, err := puts.Decode(raw); err == nil {
before = &got
}
}
var next puts.Put
switch {
case names && before != nil && before.FP == build:
continue // made already, or a person's put of the same build: it goes its own steps, ended or not
case names:
words := fmt.Sprintf("asked by %s: it names build %s of %s", placementBy, shortFP(build), key)
if before != nil && !puts.Ended(before.Step) {
words += fmt.Sprintf("; it replaces the put of %s, which stood at %s", shortFP(before.FP), before.Step)
}
next = puts.Put{FP: build, Hold: true, Why: "the placement names this build (ADR 0666 §24)", By: placementBy, At: now}.Said(now, puts.Asked, words)
case before != nil && before.By == placementBy && !puts.Ended(before.Step):
next = before.Said(now, puts.Released, fmt.Sprintf("released by %s: it names no build of its own any more; the assigner's rules write the want again", placementBy))
default:
continue
}
data, err := puts.Encode(next)
if err != nil {
p.logf("the placement's put %s is not written: %v", key, err)
continue
}
if before == nil {
_, err = kv.Create(ctx, key, data)
} else if e, gerr := kv.Get(ctx, key); gerr != nil {
err = gerr
} else {
_, err = kv.Update(ctx, key, data, e.Revision())
}
if err != nil {
p.logf("the placement's put %s could not be written, and is written at the next pass: %v", key, err)
continue
}
p.w.put(buckets.Puts(), key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
if p.acts != nil {
p.acts.say(key, next)
}
}
}
@@ -5,22 +5,15 @@ import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"mesh-delivery-assigner/internal/assign"
)
@@ -40,12 +33,6 @@ const PutEvent = "put"
type caller interface {
Call(key string, body any) (json.RawMessage, error)
CallOn(node, key string, body any) (json.RawMessage, error)
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
CallKept(key string, body any) (json.RawMessage, error)
// CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668).
CallKeptOn(node, key string, body any) (json.RawMessage, error)
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
CallFile(key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
@@ -61,7 +48,7 @@ type putActs struct {
snapNo map[string]string
warranted map[string]bool
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
rec meshhandactsclient.Recorder
rec handacts.Recorder
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
@@ -69,32 +56,32 @@ type putActs struct {
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{rec: meshhandactsclient.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p puts.Put) string {
func identity(key string, p delivery.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]puts.Put {
out := map[string]map[string]puts.Put{}
for key, value := range p.w.read(buckets.Puts()) {
node, module, err := puts.ParseKey(key)
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
}
put, err := puts.Decode(value)
put, err := delivery.DecodePut(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]puts.Put{}
out[module] = map[string]delivery.Put{}
}
out[module][node] = put
}
@@ -106,14 +93,13 @@ func (p *passer) puts(ctx context.Context) {
if p.acts == nil {
return
}
p.placementPuts(ctx)
all := p.putsByModule()
defer func() { p.putWaits(ctx, p.putsByModule()) }()
if len(all) == 0 {
return
}
assigned := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
@@ -123,34 +109,34 @@ func (p *passer) puts(ctx context.Context) {
for _, module := range modules {
var manifest catalogue.Manifest
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
if got := p.w.manifest(mod.Manifest); got.err == "" {
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
manifest = got.manifest
}
}
for node, put := range all[module] {
if puts.Ended(put.Step) {
if delivery.PutEnded(put.Step) {
continue
}
key, _ := puts.Key(node, module)
key, _ := delivery.PutKey(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) {
if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed {
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, puts.Refused, "the bus's want is not written: "+err.Error())}
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) &&
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
@@ -159,67 +145,71 @@ func (p *passer) puts(ctx context.Context) {
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted {
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !mwc.SameWant(normalWant(before), normalWant(want)) {
p.putWant(ctx, node, module, want)
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, meshdeliveryclient.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w mwc.Want) mwc.Want {
func normalWant(w delivery.Want) delivery.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].Fp2}
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
w.Build, w.BuildWords = e.Outcome, e.Error
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
if c, ok := chosen[module]; ok {
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
} else if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := asks.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey)); err == nil && a.Asking == assign.PutAsking(put) {
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
case p.answeredWith(akey, assign.OptionNo):
w.Answer = assign.OptionNo
default:
expires := assign.ExpiresAt(ask)
w.AskExpired = !expires.IsZero() && w.Now.After(expires)
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
}
}
}
}
if r, ok := p.w.runningOf(node, module); ok {
w.Running = &r
}
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
w.Have = &h
}
}
@@ -230,15 +220,15 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
var out *assign.ComposedDigest
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := grantsstate.PartGrantKey(module, fp); err == nil {
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
@@ -248,17 +238,17 @@ func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, b
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == puts.Person && next.Put.Ask != "" && w.Composed != nil {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey))
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
}
}
}
@@ -267,29 +257,29 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
}
}
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
func (p *passer) performs(key string, act occ.Act) error {
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks(), key))
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
func (p *passer) performs(key string, act asks.Act) error {
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
for _, o := range ask.Options {
for _, o := range a.Ask.Options {
if o.ID == assign.OptionYes {
return occ.Performs(o, act)
return o.Performs(act)
}
}
return fmt.Errorf("the ask %s offers no yes", key)
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b puts.Put) bool {
func samePut(a, b delivery.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put) bool {
kv := p.kvs[buckets.Puts()]
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
if kv == nil {
return false
}
@@ -298,11 +288,11 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := puts.Decode(e.Value())
now, err := delivery.DecodePut(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := puts.Encode(next)
data, err := delivery.EncodePut(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
@@ -313,7 +303,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
}
return false
}
p.w.put(buckets.Puts(), key, data)
p.w.put(delivery.AssignerPutsBucket, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
@@ -409,7 +399,7 @@ func (a *putActs) warrant(id, ask string) {
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
@@ -438,8 +428,8 @@ type putSaid struct {
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p puts.Put) {
node, module, err := puts.ParseKey(key)
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
if err != nil || a.bus == nil {
return
}
@@ -1,116 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its
// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint
// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release,
// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none.
// putWaitBound is the bound of a put without hold's wait.
const putWaitBound = 30 * time.Minute
// Wait-ended's how of a put's wait.
const (
putWaitRan = "ran"
putWaitReleased = "released"
putWaitSuperseded = "superseded"
)
// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run.
func putWaits(p puts.Put) bool {
return !p.Hold && p.Step == puts.Wanted
}
// putWaitBody is the wait of the put p of module on node.
func putWaitBody(node, module string, p puts.Put) meshwants.Wait {
since := p.StandsSince().UTC()
return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module(), Node: node, Module: module, Fingerprint: p.FP,
Since: since, ClockFrom: &since, Bound: putWaitBound.String(),
Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)}
}
// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204,
// A-211): its identity and the put as it was then.
type putWaiting struct {
ID string `json:"id"`
Put puts.Put `json:"put"`
}
// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left.
func (p *passer) saidPutWaits() map[string]putWaiting {
out := map[string]putWaiting{}
for key, value := range p.w.read(buckets.PutWaits()) {
var w putWaiting
if err := json.Unmarshal(value, &w); err != nil || w.ID == "" {
p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits(), key, err)
continue
}
out[key] = w
}
return out
}
// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended
// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said;
// one not said, or not kept, is said at the next pass. Without the state no put's wait is said.
func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) {
if p.kvs[buckets.PutWaits()] == nil {
return
}
waiting := p.saidPutWaits()
for key, was := range waiting {
node, module, err := puts.ParseKey(key)
if err != nil {
continue
}
cur, there := all[module][node]
same := there && identity(key, cur) == was.ID
if same && putWaits(cur) {
continue
}
how := putWaitSuperseded
if same && cur.Step == puts.Done {
how = putWaitRan
} else if same && cur.Step == puts.Released {
how = putWaitReleased
}
body := putWaitBody(node, module, was.Put)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil {
p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
p.remove(ctx, buckets.PutWaits(), key)
delete(waiting, key)
}
for module, byNode := range all {
for node, put := range byNode {
key, err := puts.Key(node, module)
if err != nil || !putWaits(put) {
continue
}
id := identity(key, put)
if w, said := waiting[key]; said && w.ID == id {
continue
}
if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil {
p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
kept := putWaiting{ID: id, Put: put}
p.put(ctx, buckets.PutWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
}
}
}
@@ -1,92 +0,0 @@
package main
import (
"context"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as
// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is
// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said
// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one
// that ran while the assigner was stopped gets its wait-ended at the next start.
func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) {
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
kept := &memKV{values: map[string][]byte{}}
var events []string
var bodies []meshwants.Wait
start := func() *passer { // a process's start: a fresh world that hands what the state holds
p := newPasser(kvSet{buckets.PutWaits(): kept}, at)
for k, v := range kept.values {
p.w.put(buckets.PutWaits(), k, v)
}
p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {})
p.publishWants = func(event string, body any, id string) error {
events, bodies = append(events, event), append(bodies, body.(meshwants.Wait))
return nil
}
return p
}
ctx := context.Background()
asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked")
wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed")
held := wanted
held.Hold = true
p := start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil {
t.Fatalf("a put that waits said %v, kept %q", events, kept.values)
}
events, bodies = nil, nil
// At the next start the put still waits: it was said before, nothing is said of it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 0 {
t.Fatalf("a put waiting at the start was said again: %v", events)
}
// It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}})
if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut ||
kept.values["ace.nats"] != nil {
t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values)
}
events, bodies = nil, nil
// A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing.
again := wanted
again.At = at.Add(time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("a put's written want said %v", events)
}
b := bodies[0]
if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil ||
!b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" {
t.Errorf("the put's wait is %+v", b)
}
// Said once.
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 {
t.Errorf("a put's wait was said again: %v", events)
}
// Put again under its key: the first wait ends superseded, the new one is held.
newer := asked
newer.At = at.Add(2 * time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}})
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest {
t.Errorf("a put asked again said %v %+v", events, bodies)
}
// The answer of assigner_puts.
if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" {
t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound)
}
if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" {
t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound)
}
}
@@ -1,177 +0,0 @@
package main
import (
"context"
"strconv"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
var revertReasons = []string{"restore point missing", "restore point damaged"}
// revertReason is the reason of a have's words when they say a held revert, or "".
func revertReason(words string) string {
for _, r := range revertReasons {
if strings.HasPrefix(words, r) {
return r
}
}
return ""
}
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
// that holds a revert, which backups leaves to it.
func (p *passer) reverts(ctx context.Context) map[string]bool {
taken := map[string]bool{}
wanted := p.w.wantsOf()
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := nodeengineclient.DecodeHave(value)
if err != nil {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := asks.RevertAskKey(node, module, want.Fp)
if err != nil {
continue
}
reason := revertReason(have.Detail)
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
if have.Fp == want.Fp && p.w.has(buckets.Asks(), akey) {
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
}
continue
}
taken[node+"."+module] = true
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
if !ok {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
if there != nil {
now := p.clock()
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
continue
}
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
continue // not answered, or answered hold: the hold stands and nothing is written
}
words := assign.RevertAnswerWords(answer.Option, current.Asking)
if want.Revert != mwc.RevertLive || want.Words != words {
want.Revert, want.Words = mwc.RevertLive, words
p.putWant(ctx, node, module, want)
}
}
return taken
}
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
if p.graph == nil || p.entries == nil {
return "", "", false
}
run, heard := p.w.runningOf(node, module)
if !heard || run.Stopped || run.Fingerprint == "" {
return "", "", false
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
return "", "", false
}
if !reverted {
return "", "", false
}
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
if !known {
return "", "", false // asked: its answer wakes a pass
}
var names, forms []string
for _, c := range changes {
f := 0
if c.Applied != nil {
f = c.Applied.Form
}
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
}
if len(names) == 0 {
return "", "", false
}
return strings.Join(names, ", "), strings.Join(forms, ", "), true
}
// revertFinding writes the finding revert-held, or clears it, once per process each way.
func (p *passer) revertFinding(o assign.Observed, raised bool) {
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
said := key + ".raised"
if !raised {
said = key + ".cleared"
}
if p.said[said] {
return
}
var err error
switch {
case raised && p.findings.Put != nil:
err = p.findings.Put(o.Finding(p.clock()))
case !raised && p.findings.Delete != nil:
err = p.findings.Delete(key)
default:
return
}
if err != nil {
p.logf("the finding %s is not written: %v", key, err)
return
}
p.said[said] = true
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
}
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
if p.entries == nil {
return nil, false
}
running, rs := p.entries.Need(ctx, runFp)
applied, as := p.entries.Need(ctx, wantFp)
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
return nil, false
}
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
}
changes, err := assigngraph.FormChanges(running, applied)
if err != nil {
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
return nil, false
}
return changes, true
}
@@ -1,136 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, at)
p.logf = t.Logf
findings := map[string][]byte{}
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
objects := memObjects{}
for i, fp := range []string{fpOld, fpTest} {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
entry := func(fp string, form int) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
}
answered := make(chan struct{}, 4)
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
wkey, _ := mwc.WantKey("ace", "web")
want := mwc.Want{Fp: fpOld, Self: true}
raw, _ := mwc.EncodeWant(want)
p.w.put(buckets.Wants(), wkey, raw)
hkey, _ := nodeengineclient.HaveKey("ace", "web")
have := func(fp, phase, detail string) {
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
}
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
pass := func() { p.backups(ctx, p.reverts(ctx)) }
pass() // the entries are asked; nothing is asked of the operator until they answer
for range 2 {
select {
case <-answered:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
}
}
pass()
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
if !p.w.has(buckets.Asks(), rkey) || p.w.has(buckets.Asks(), bkey) {
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
}
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks())[rkey])
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
t.Fatalf("the revert ask = %+v, %v", ask, err)
}
fkey := "delivery.ace.web.revert-held"
if _, raised := findings[fkey]; !raised {
t.Fatalf("findings %v: want %s", findings, fkey)
}
answer := func(option string) {
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
p.w.put(buckets.Answers, rkey, v)
}
puts := wantsKV.puts
answer(assign.OptionRevertHold)
pass()
if wantsKV.puts != puts {
t.Fatal("an answer hold wrote a want")
}
answer(assign.OptionRevertLive)
pass()
got, err := mwc.DecodeWant(p.w.read(buckets.Wants())[wkey])
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
t.Fatalf("the want after live = %+v, %v", got, err)
}
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
pass()
if _, raised := findings[fkey]; raised {
t.Fatal("the finding stands while the reverted build runs")
}
// A newer want replaces the held revert: the ask goes and the finding with it.
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
pass()
if _, raised := findings[fkey]; !raised {
t.Fatal("the finding is not raised again for a hold again")
}
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
p.w.put(buckets.Wants(), wkey, raw)
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
}
}
@@ -1,88 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running.<node>.<module>` is
// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every
// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it.
// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one.
func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
if err := w.hearRunning(m.Subject(), m.Data()); err != nil {
logf("%v", err)
return
}
wake()
}, logf)
if err != nil {
nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err)
return
}
w.mu.Lock()
w.runningHeard = true
w.mu.Unlock()
nh.Heard(nodeengine.StreamRunning)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearRunning keeps one `running` event said on subject.
func (w *world) hearRunning(subject string, data []byte) error {
var r nodeengine.Running
if err := json.Unmarshal(data, &r); err != nil {
return fmt.Errorf("%s holds what is not a running: %w", subject, err)
}
if r.Node == "" || r.Module == "" {
return fmt.Errorf("%s names no node or no module, so it is kept for none", subject)
}
w.mu.Lock()
w.running[r.Node+"."+r.Module] = r
w.mu.Unlock()
return nil
}
// runningOf is the last `running` heard of module on node.
func (w *world) runningOf(node, module string) (nodeengine.Running, bool) {
w.mu.Lock()
defer w.mu.Unlock()
r, ok := w.running[node+"."+module]
return r, ok
}
// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the
// have).
func (w *world) runningNow() map[string]map[string]nodeengine.Running {
w.mu.Lock()
defer w.mu.Unlock()
if !w.runningHeard {
return nil
}
out := map[string]map[string]nodeengine.Running{}
for _, r := range w.running {
if out[r.Node] == nil {
out[r.Node] = map[string]nodeengine.Running{}
}
out[r.Node][r.Module] = r
}
return out
}
@@ -6,58 +6,46 @@ import (
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"reflect"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mdcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/judge"
)
// The seat mesh-module-assigner's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
// proposal one step, from what the world holds. A proposal the verbs wrote is said (assignments-proposed); the nodes it
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
// candidate; their answers are judged (assignments-judged) and, on a passing verdict, the operator is asked to approve it;
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (assignments-changed),
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
// lays the proposal again; a decline, an expiry or a withdrawal ends it (assignments-withdrawn). Every event is said on the
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
type publisher func(event string, body any, id string) error
// seatPublisher publishes the seat's events on mesh.seat.mesh-module-assigner.event.<event>, the seat's own subject, which
// the shared events stream keeps and every consumer of `mesh-module-assigner.<event>` hears (the seat's family, 7 days),
// with the headers an emit carries.
func seatPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatdef.EventSubject(mma.Seat, event))
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", assignment.Module())
msg.Header.Set("x-source", delivery.AssignerModule)
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
@@ -68,32 +56,32 @@ func seatPublisher(ctx context.Context, js jetstream.JetStream, node string) pub
}
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
// the controller: assignments is the one source (ADR 0567 §8, issue 774).
// the controller: placement is the one source (ADR 0567 §8, issue 774).
type seatActs struct {
bus caller
wake func()
logf func(string, ...any)
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
rec meshhandactsclient.Recorder
rec handacts.Recorder
}
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
return &seatActs{bus: bus, wake: wake, logf: logf, rec: meshhandactsclient.Recorder{Ask: bus.Call}}
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
}
// record records once, in the hand-act log, that assignments was written on the operator's warrant (§5): the router's
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
// the controller's while that seat has no holder (ADR 0565 §11, meshhandactsclient.Recorder).
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
func (a *seatActs) record(warrant string) {
if a == nil || warrant == "" {
return
}
ask, _, _ := strings.Cut(warrant, "@")
go func() {
if taken, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
a.logf("the assignments's warrant %s is not recorded in the hand-act log: %v", ask, err)
if taken, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
} else {
a.logf("the assignments's warrant %s is recorded in the log of %s", ask, taken.Log)
a.logf("the placement's warrant %s is recorded in the log of %s", ask, taken.Log)
}
}()
}
@@ -136,7 +124,7 @@ func (p *passer) sayWaits(unbound []string) {
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
return
}
if c.State == occ.StateCleared && p.saidWaits == "" {
if c.State == asks.StateCleared && p.saidWaits == "" {
return // nothing was raised by this process
}
if err := p.say(c); err != nil {
@@ -146,25 +134,26 @@ func (p *passer) sayWaits(unbound []string) {
p.said[c.Key], p.saidWaits = true, state
}
// seatAssignments is every node's assignments and the revision of its node key, as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
func (p *passer) seatAssignments() (map[string]mma.Assignments, map[string]uint64) {
return p.w.assignmentKeys.AllRevisions()
// seatPlacements is every node's placement and the revision of its node key (its legacy key's for a node not yet
// converted), as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
return p.w.placement.AllRevisions()
}
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
func (p *passer) candidatesOf(id string) (map[string]mma.CandidatesByProposalNodeValue, map[string]uint64, time.Time) {
out, revs := map[string]mma.CandidatesByProposalNodeValue{}, map[string]uint64{}
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
var newest time.Time
for key, value := range p.w.read(plc.CandidatesBucket()) {
pid, node, err := plc.ParseCandidateKey(key)
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
pid, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil || pid != id {
continue
}
c, err := plc.DecodeCandidate(value)
c, err := delivery.DecodePlacementCandidate(value)
if err != nil {
continue
}
meta := p.w.metaOf(plc.CandidatesBucket(), key)
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
out[node], revs[node] = c, meta.revision
if meta.at.After(newest) {
newest = meta.at
@@ -174,14 +163,14 @@ func (p *passer) candidatesOf(id string) (map[string]mma.CandidatesByProposalNod
}
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
func (p *passer) answersTo(id string) map[string]*nodeengine.CandidateAnswer {
out := map[string]*nodeengine.CandidateAnswer{}
for key, value := range p.w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range p.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := nodeengineclient.DecodeHaveNode(value)
h, err := delivery.DecodeHaveNode(value)
if err != nil {
continue
}
@@ -192,9 +181,9 @@ func (p *passer) answersTo(id string) map[string]*nodeengine.CandidateAnswer {
return out
}
// assignmentsSeat takes every open proposal one step further.
func (p *passer) assignmentsSeat(ctx context.Context) {
for _, b := range seatBuckets() {
// placementSeat takes every open proposal one step further.
func (p *passer) placementSeat(ctx context.Context) {
for _, b := range seatBuckets {
if p.kvs[b] == nil {
return
}
@@ -202,77 +191,70 @@ func (p *passer) assignmentsSeat(ctx context.Context) {
if p.saidProposed == nil {
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
}
if !p.copyRetired(ctx) {
return // the old state is copied whole before the seat takes a step: a half-copied state would place nothing on a node
}
current, revs := p.seatAssignments()
p.convertPlacement(ctx)
current, revs := p.seatPlacements()
ids := make([]string, 0)
raws := p.w.read(plc.ProposalsBucket())
raws := p.w.read(delivery.PlacementProposalsBucket)
for id := range raws {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
prop, err := plc.DecodeProposal(raws[id])
prop, err := delivery.DecodePlacementProposal(raws[id])
if err != nil || prop.ID != id {
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
continue
}
p.stepProposal(ctx, prop, p.w.metaOf(plc.ProposalsBucket(), id).revision, current, revs)
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
}
}
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
kv := p.kvs[plc.PulsesBucket()]
kv := p.kvs[delivery.PlacementPulsesBucket]
_ = kv.Delete(ctx, key)
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
if err != nil {
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
return
}
p.w.putEntry(plc.PulsesBucket(), key, []byte(`{}`), rev, p.clock())
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
}
// pulsing says whether a bound's key stands.
func (p *passer) pulsing(key string) bool { return p.w.has(plc.PulsesBucket(), key) }
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
// deletePulses deletes every bound of a proposal.
func (p *passer) deletePulses(ctx context.Context, id string) {
for key := range p.w.read(plc.PulsesBucket()) {
for key := range p.w.read(delivery.PlacementPulsesBucket) {
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
if err := p.kvs[plc.PulsesBucket()].Delete(ctx, key); err == nil {
p.w.put(plc.PulsesBucket(), key, nil)
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
p.w.put(delivery.PlacementPulsesBucket, key, nil)
}
}
}
}
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string) (mma.Proposal, bool) {
a, err := assign.AssignmentsAsk(akey, prop, brief, assign.AssignmentsAsking(prop, p.clock()), p.clock())
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
if err != nil {
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
placed, err := assign.AskOf(a)
if err != nil {
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
}
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
if pk, err := plc.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
}
prop.State, prop.Ask = mmac.ProposalAsked, placed.ID
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
return prop, true
}
// stepProposal takes one proposal one step further.
func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64) {
if !p.saidProposed[prop.ID] && prop.State == mmac.ProposalAnswering && prop.Verdict == nil {
p.sayEvent(mma.EventAssignmentsProposed, mma.AssignmentsEvent{Proposal: prop}, prop.ID)
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
p.saidProposed[prop.ID] = true
}
if prop.Withdrawn != "" {
@@ -281,17 +263,17 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
}
if prop.Seed || prop.HandBack {
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: assignments is the mesh's alone")
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
return
}
if prop.Warrant != nil {
p.addNode(ctx, prop, revs)
return
}
akey, _ := asks.AssignmentsAskKey(prop.ID)
again := prop.Again != nil && !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == mmac.ProposalAsked {
askPulse, _ := plc.PulseAskKey(prop.ID)
akey, _ := delivery.PlacementAskKey(prop.ID)
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == delivery.ProposalAsked {
askPulse, _ := delivery.PulseAskKey(prop.ID)
switch {
case p.answeredWith(akey, assign.OptionYes):
p.warranted(ctx, prop, rev, akey, current, revs)
@@ -310,13 +292,6 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
}
cands, candRevs, written := p.candidatesOf(prop.ID)
if len(prop.Must) == 0 {
for _, node := range plc.Nodes(prop) {
if _, ok := cands[node]; !ok && !p.laidHere[prop.ID] {
// The verb writes the proposal alone: its candidates have one writer, this process (design 47 G A-317).
p.lay(ctx, prop, rev, current, revs, prop.Words, true)
return
}
}
p.mustAnswer(ctx, prop, rev, current, cands)
return
}
@@ -325,12 +300,12 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
return // a candidate just written is not heard yet: its watch wakes the next pass
}
}
if !written.IsZero() && (prop.Written == nil || written.After(*prop.Written)) {
prop.Written = &written
if written.After(prop.Written) {
prop.Written = written
}
pulsing := map[string]bool{}
for _, node := range prop.Must {
if k, err := plc.PulseAnswerKey(prop.ID, node); err == nil {
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
pulsing[node] = p.pulsing(k)
}
}
@@ -340,12 +315,12 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
}
prop.Verdict = &v
switch v.State {
case mdcc.VerdictFail:
prop.State, prop.Ask = mmac.ProposalFailed, ""
case plc.VerdictWaits:
prop.State, prop.Ask = mmac.ProposalAnswering, ""
case mdcc.VerdictPass:
if prop.State != mmac.ProposalAsked || again {
case delivery.VerdictFail:
prop.State, prop.Ask = delivery.ProposalFailed, ""
case delivery.VerdictWaits:
prop.State, prop.Ask = delivery.ProposalAnswering, ""
case delivery.VerdictPass:
if prop.State != delivery.ProposalAsked || again {
var ok bool
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
return
@@ -356,21 +331,21 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
}
}
if p.writeProposal(ctx, prop, rev) {
p.sayEvent(mma.EventAssignmentsJudged, mma.AssignmentsEvent{Proposal: prop},
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
}
}
// sameVerdict says two verdicts say the same, their time aside.
func sameVerdict(a, b mma.Verdict) bool {
func sameVerdict(a, b delivery.PlacementVerdict) bool {
a.At, b.At = time.Time{}, time.Time{}
return reflect.DeepEqual(a, b)
}
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
func moved(prop mma.Proposal, revs map[string]uint64) bool {
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
for node, base := range prop.Base {
if revs[node] != uint64(base) {
if revs[node] != base {
return true
}
}
@@ -378,7 +353,7 @@ func moved(prop mma.Proposal, revs map[string]uint64) bool {
}
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
func (p *passer) intendedOf(prop mma.Proposal, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) map[string]map[string]bool {
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
key := fmt.Sprint(prop.ID, prop.Base)
if got, ok := p.intended[key]; ok {
return got
@@ -388,14 +363,14 @@ func (p *passer) intendedOf(prop mma.Proposal, current map[string]mma.Assignment
return got
}
// compute is what the proposal changes on each node by its assignments, and the nodes that answer it: every node with a
// assignments or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
// compute is what the proposal changes on each node by its placement, and the nodes that answer it: every node with a
// placement or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
// change in its answer.
func (p *passer) compute(prop mma.Proposal, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) (map[string]map[string]bool, []string) {
laid := map[string]*mma.Assignments{}
for _, node := range plc.Nodes(prop) {
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string) {
laid := map[string]*delivery.Placement{}
for _, node := range prop.Nodes() {
if c, ok := cands[node]; ok && !c.Reached {
laid[node] = c.Assignments
laid[node] = c.Placement
}
}
nodes := map[string]bool{}
@@ -410,8 +385,8 @@ func (p *passer) compute(prop mma.Proposal, current map[string]mma.Assignments,
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
// change, and writes them into the proposal.
func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) {
for _, node := range plc.Nodes(prop) {
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
for _, node := range prop.Nodes() {
if _, ok := cands[node]; !ok {
return // the verbs write the candidates before the proposal: wait to hear them
}
@@ -422,18 +397,18 @@ func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64,
if _, ok := cands[node]; ok {
continue
}
key, err := plc.CandidateKey(prop.ID, node)
key, err := delivery.PlacementCandidateKey(prop.ID, node)
if err != nil {
continue
}
c := mma.CandidatesByProposalNodeValue{Proposal: prop.ID, Reached: true}
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
if pl, ok := current[node]; ok {
c.Assignments = &pl
c.Placement = &pl
}
p.put(ctx, plc.CandidatesBucket(), key, func() ([]byte, error) { return plc.EncodeCandidate(c) })
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
for _, node := range must {
if k, err := plc.PulseAnswerKey(prop.ID, node); err == nil {
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
p.pulse(ctx, k, judge.AnswerBound)
}
}
@@ -443,40 +418,22 @@ func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64,
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string) {
p.lay(ctx, prop, rev, current, revs, why, false)
}
// lay lays a proposal over the keys as they stand now and writes its candidates, each with what it adds (A-316); first says it is the
// proposal's first laying, the verb having written the proposal alone (A-317).
func (p *passer) lay(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string, first bool) {
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
held, recorded := p.lays(ctx)
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.Heard()})
if slices.ContainsFunc(findings, plc.WaitsForSeatList) {
// ADR 0636 §1: nothing is ended on a list not read; the reader's change makes a pass, which lays it again.
p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.Health())
return
}
Current: current, Revisions: revs, Held: held, Recorded: recorded})
if len(findings) > 0 {
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
return
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
if p.laidHere == nil {
p.laidHere = map[string]bool{}
}
p.laidHere[prop.ID] = true
for _, node := range plc.Nodes(again) {
key, _ := plc.CandidateKey(prop.ID, node)
c := p.candidateOf(prop.ID, node, laid[node], current)
p.put(ctx, plc.CandidatesBucket(), key, func() ([]byte, error) { return plc.EncodeCandidate(c) })
}
again.Written, again.Words = ptr(p.clock().UTC()), why+": laid again over the keys as they stand, to be judged and asked again"
if first {
again.Words = why
for _, node := range again.Nodes() {
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
p.writeProposal(ctx, again, rev)
}
@@ -484,17 +441,19 @@ func (p *passer) lay(ctx context.Context, prop mma.Proposal, rev uint64, current
// nodes with a record.
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
held := map[string]bool{}
for name := range p.w.trunk() {
held[name] = true
for key := range p.w.read(delivery.ModulesBucket) {
if name, err := delivery.ParseModulesKey(key); err == nil {
held[name] = true
}
}
records := p.w.records()
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
}
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
// proposal deleted, and assignments-changed said.
func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, akey string, current map[string]mma.Assignments, revs map[string]uint64) {
if err := p.performs(akey, assign.AssignmentsAct(prop)); err != nil {
// proposal deleted, and placement-changed said.
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
return
}
@@ -504,17 +463,17 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
return
}
cands, _, _ := p.candidatesOf(prop.ID)
laid := map[string]*mma.Assignments{}
for _, node := range plc.Nodes(prop) {
laid[node] = cands[node].Assignments
laid := map[string]*delivery.Placement{}
for _, node := range prop.Nodes() {
laid[node] = cands[node].Placement
}
writes, err := assign.Writes(prop, laid)
if err != nil {
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
return
}
var changes []plc.Change
writtenLaid := map[string]*mma.Assignments{}
var changes []delivery.PlacementChange
writtenLaid := map[string]*delivery.Placement{}
after := maps.Clone(current)
afterRevs := maps.Clone(revs)
for _, w := range writes {
@@ -522,7 +481,7 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
if err != nil {
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
// again over the keys as this write left them.
p.logf("assignments %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
p.followWrite(prop, answer, changes)
if rest, ok := assign.Rest(prop, writtenLaid); ok {
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
@@ -532,7 +491,7 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
return
}
before := current[w.Node]
now := mma.Assignments{Node: w.Node}
now := delivery.Placement{Node: w.Node}
if w.Value != nil {
now = *w.Value
after[w.Node] = *w.Value
@@ -540,32 +499,32 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
delete(after, w.Node)
}
afterRevs[w.Node] = rev2
c := plc.ChangeOf(before, now)
c := delivery.ChangeOf(before, now)
c.Before, c.After = w.Base, rev2
changes = append(changes, c)
writtenLaid[w.Node] = w.Value
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.followWrite(prop, answer, changes)
}
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
// not record it again; assignments-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
func (p *passer) addNode(ctx context.Context, prop mma.Proposal, revs map[string]uint64) {
current, _ := p.seatAssignments()
// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) {
current, _ := p.seatPlacements()
if moved(prop, revs) {
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
return
}
laid, findings := plc.LayChanges(current, prop.Changes)
laid, findings := delivery.LayChanges(current, prop.Changes)
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
node := prop.Changes[0].Node
if prop.Changes[0].Op == mmac.OpRemoveNode {
if prop.Changes[0].Op == delivery.OpRemoveNode {
p.removeNode(ctx, prop, current, findings)
return
}
@@ -579,22 +538,22 @@ func (p *passer) addNode(ctx context.Context, prop mma.Proposal, revs map[string
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
return
}
c := plc.ChangeOf(mma.Assignments{Node: node}, value)
c.Before, c.After = uint64(prop.Base[node]), rev
c := delivery.ChangeOf(delivery.Placement{Node: node}, value)
c.Before, c.After = prop.Base[node], rev
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
w := prop.Warrant
p.sayChanged(prop, mdc.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
[]plc.Change{c})
[]delivery.PlacementChange{c})
}
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; assignments-changed carries the warrant.
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant.
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
func (p *passer) removeNode(ctx context.Context, prop mma.Proposal, current map[string]mma.Assignments, findings []string) {
func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) {
node := prop.Changes[0].Node
before, ok := current[node]
if len(findings) > 0 || !ok {
@@ -606,21 +565,21 @@ func (p *passer) removeNode(ctx context.Context, prop mma.Proposal, current map[
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
return
}
c := plc.ChangeOf(before, mma.Assignments{Node: node})
c.Before, c.After = uint64(prop.Base[node]), rev
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
c.Before, c.After = prop.Base[node], rev
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
w := prop.Warrant
p.sayChanged(prop, mdc.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
Words: fmt.Sprintf("the machine %s left assignments: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
[]plc.Change{c})
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
[]delivery.PlacementChange{c})
}
// followWrite follows the keys a warrant wrote: the warrant recorded and assignments-changed said. No verb of the
// controller is called: the assignments is the one source of where modules run (ADR 0567 §8, issue 774).
func (p *passer) followWrite(prop mma.Proposal, answer mdc.Answer, changes []plc.Change) {
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
if len(changes) == 0 {
return
}
@@ -628,7 +587,7 @@ func (p *passer) followWrite(prop mma.Proposal, answer mdc.Answer, changes []plc
p.sayChanged(prop, answer, changes)
}
func sortedNodes(m map[string]*mma.Assignments) []string {
func sortedNodes(m map[string]*delivery.Placement) []string {
out := make([]string, 0, len(m))
for n := range m {
out = append(out, n)
@@ -637,8 +596,8 @@ func sortedNodes(m map[string]*mma.Assignments) []string {
return out
}
// sayChanged says assignments-changed for the keys written.
func (p *passer) sayChanged(prop mma.Proposal, answer mdc.Answer, changes []plc.Change) {
// sayChanged says placement-changed for the keys written.
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
if len(changes) == 0 {
return
}
@@ -648,36 +607,36 @@ func (p *passer) sayChanged(prop mma.Proposal, answer mdc.Answer, changes []plc.
at = t
}
}
plc.SortChanges(changes)
p.sayEvent(mma.EventAssignmentsChanged, mma.AssignmentsEvent{Proposal: prop, Nodes: changeBodies(changes),
Warrant: &mma.AssignmentsEventWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
delivery.SortChanges(changes)
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
}
// endProposal ends a proposal: its ask, its candidates and its key deleted, and assignments-withdrawn said carrying it
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop mma.Proposal, why string) {
if akey, err := asks.AssignmentsAskKey(prop.ID); err == nil && p.w.has(buckets.Asks(), akey) {
if err := p.kvs[buckets.Asks()].Delete(ctx, akey); err == nil {
p.w.put(buckets.Asks(), akey, nil)
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
p.w.put(delivery.AssignerAsksBucket, akey, nil)
}
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
if err := p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
return
}
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.logf("the proposal %s ends: %s", prop.ID, why)
p.sayEvent(mma.EventAssignmentsWithdrawn, mma.AssignmentsEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
}
// deleteCandidates deletes every candidate of a proposal.
func (p *passer) deleteCandidates(ctx context.Context, id string) {
for key := range p.w.read(plc.CandidatesBucket()) {
if pid, _, err := plc.ParseCandidateKey(key); err == nil && pid == id {
if err := p.kvs[plc.CandidatesBucket()].Delete(ctx, key); err == nil {
p.w.put(plc.CandidatesBucket(), key, nil)
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
}
}
}
@@ -685,13 +644,13 @@ func (p *passer) deleteCandidates(ctx context.Context, id string) {
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
func (p *passer) writeProposal(ctx context.Context, prop mma.Proposal, rev uint64) bool {
raw, err := plc.EncodeProposal(prop)
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
raw, err := delivery.EncodePlacementProposal(prop)
if err != nil {
p.logf("the proposal %s is not written: %v", prop.ID, err)
return false
}
kv := p.kvs[plc.ProposalsBucket()]
kv := p.kvs[delivery.PlacementProposalsBucket]
var at uint64
if rev == 0 {
at, err = kv.Create(ctx, prop.ID, raw)
@@ -704,12 +663,12 @@ func (p *passer) writeProposal(ctx context.Context, prop mma.Proposal, rev uint6
}
return false
}
p.w.putEntry(plc.ProposalsBucket(), prop.ID, raw, at, p.clock())
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
return true
}
// say says one of the seat's events, said in the journal when the bus refuses it.
func (p *passer) sayEvent(event string, body mma.AssignmentsEvent, id string) {
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
if p.publish == nil {
return
}
@@ -718,30 +677,30 @@ func (p *passer) sayEvent(event string, body mma.AssignmentsEvent, id string) {
}
}
// writeNode writes one node's assignments after (nil: none) as its keys (mesh-sdk mmac.Keys.Writes; research
// writeNode writes one node's placement after (nil: none) as its keys (mesh-sdk delivery.PlacementKeys.Writes; research
// 072 row 37): the module keys that change, each carrying the value it replaces, then the node key, the commit, then the
// module keys no longer placed. It answers the node key's revision after (0: deleted). A write that fails up to the
// commit ends it with nothing changed for any reader, the module keys written before it read at their previous values;
// a delete after the commit that fails leaves a module key the node key does not name, which places nothing and is
// deleted at the node's next write. Each key written is taken into the world at once, so the next write of the pass
// reads it.
func (p *passer) writeNode(ctx context.Context, node string, after *mma.Assignments) (uint64, error) {
writes, err := p.w.assignmentKeys.Writes(node, after)
func (p *passer) writeNode(ctx context.Context, node string, after *delivery.Placement) (uint64, error) {
writes, err := p.w.placement.Writes(node, after)
if err != nil {
return 0, err
}
kv := p.kvs[plc.AssignmentsBucket()]
kv := p.kvs[delivery.PlacementBucket]
committed, rev := false, uint64(0)
for _, w := range writes {
got, err := writeKey(ctx, kv, w)
if err != nil {
if committed {
p.logf("assignments %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
p.logf("placement %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
continue
}
return 0, fmt.Errorf("assignments %s at revision %d: %w", w.Key, w.Base, err)
return 0, fmt.Errorf("placement %s at revision %d: %w", w.Key, w.Base, err)
}
p.w.putEntry(plc.AssignmentsBucket(), w.Key, w.Value, got, p.clock())
p.w.putEntry(delivery.PlacementBucket, w.Key, w.Value, got, p.clock())
if w.Commit {
committed, rev = true, got
}
@@ -749,8 +708,8 @@ func (p *passer) writeNode(ctx context.Context, node string, after *mma.Assignme
return rev, nil
}
// writeKey makes one write of an assignments key by compare-and-set and answers the key's revision after (0: deleted).
func writeKey(ctx context.Context, kv jetstream.KeyValue, w mmac.KeyWrite) (uint64, error) {
// writeKey makes one write of a placement key by compare-and-set and answers the key's revision after (0: deleted).
func writeKey(ctx context.Context, kv jetstream.KeyValue, w delivery.PlacementKeyWrite) (uint64, error) {
switch {
case w.Value == nil:
return 0, kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
@@ -761,13 +720,29 @@ func writeKey(ctx context.Context, kv jetstream.KeyValue, w mmac.KeyWrite) (uint
}
}
func ptr[T any](v T) *T { return &v }
// changeBodies are the changes as the seat's event carries them.
func changeBodies(changes []plc.Change) []map[string]any {
out := make([]map[string]any, 0, len(changes))
for _, c := range changes {
out = append(out, c.Body())
// convertPlacement converts every node the state holds only in the form before, `placement.<node>`, into its node key
// and module keys (research 072 row 37), once: each node converted is said, and a node whose conversion fails is said
// and tried again at the next pass. The legacy key is left for the readers not yet delivered; a node key wins over it
// for every reader that reads both, and it is deleted with the node key when the node's placement is.
func (p *passer) convertPlacement(ctx context.Context) {
if p.convertedAll {
return
}
return out
migrations, err := p.w.placement.Migrations()
if err != nil {
p.logf("placement: some nodes in the form before cannot be converted, and stay read from it: %v", err)
}
failed := false
for _, node := range slices.Sorted(maps.Keys(migrations)) {
before, rev, _ := p.w.placement.Of(node)
got, err := p.writeNode(ctx, node, &before)
if err != nil {
failed = true
p.logf("placement: %s is not converted from placement.%s (revision %d) to its node and module keys, and is tried again at the next pass: %v", node, node, rev, err)
continue
}
p.logf("placement: %s converted from placement.%s (revision %d) to %s.node (revision %d) and %d module keys (research 072 row 37); placement.%s stays for readers not yet delivered",
node, node, rev, node, got, len(before.Modules), node)
}
p.convertedAll = !failed && err == nil
}
@@ -1,32 +1,22 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// The seat mesh-module-assigner's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
@@ -46,77 +36,46 @@ func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
}
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
// `<module>.<state>` where the manifest declares them under `reads`.
const haveState = "node-engine.have"
// The indexer's states the verbs read, each asked of the indexer's first-view tool of that name, never read from its bus
// state (novox/hq ADR 0596; ADR 0615 §1, ADR 0621 §1, §2: the manifest invokes them, and the indexer answers them to the
// assigner alone of the delivery's parts).
// `<module>.<state>`.
const (
indexerModules = "modules"
indexerManifests = "manifests"
modulesState = delivery.IndexerModule + ".modules"
haveState = "node-engine.have"
)
// indexerView is one of the indexer's states whole, asked of its first-view tool through ask (stdio.Ask, or a test's) page
// after page, each page continued after the one before's next. It is asked again at every call: the indexer's state is
// the truth, and the verbs keep no copy of it.
func indexerView(ask func(string, any) (json.RawMessage, error), state string) (map[string]json.RawMessage, error) {
tool := buckets.IndexerModule + "." + state
if ask == nil {
return nil, fmt.Errorf("%s cannot be asked here", tool)
}
page := views.AskVia(ask, tool)
out := map[string]json.RawMessage{}
for after := ""; ; {
a, err := page(context.Background(), after)
if err != nil {
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
}
maps.Copy(out, a.Values)
if a.Next == "" || a.Next == after {
return out, nil
}
after = a.Next
}
}
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
type seatTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
now func() time.Time
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatlist.Part
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
return &seatTools{st: st, ask: ask, now: time.Now}
}
// keyed is one assignments key as a verb answers it.
// keyed is one placement key as a verb answers it.
type keyed struct {
Assignments mma.Assignments `json:"assignments"`
Revision uint64 `json:"revision"`
Placement delivery.Placement `json:"placement"`
Revision uint64 `json:"revision"`
}
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
keys, err := s.st.keys(mma.StateAssignments)
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
keys, err := s.st.keys(delivery.PlacementState)
if err != nil {
return nil, nil, fmt.Errorf("assignments cannot be listed: %w", err)
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
}
sort.Strings(keys)
held := mmac.NewKeys()
held := delivery.NewPlacementKeys()
var bad []string
for _, key := range keys {
if _, _, _, err := mmac.ParseKey(key); err != nil {
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
continue
}
raw, rev, found, err := s.st.get(mma.StateAssignments, key)
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
if err != nil {
return nil, nil, fmt.Errorf("assignments key %s cannot be read: %w", key, err)
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
}
if !found {
continue
@@ -127,21 +86,20 @@ func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64
}
out, revs := held.AllRevisions()
if len(bad) > 0 && len(out) == 0 {
return nil, nil, fmt.Errorf("assignments cannot be read: %s", strings.Join(bad, "; "))
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
}
return out, revs, nil
}
// lays is what a proposal is checked against: the modules the indexer holds, as its first view of modules answers them,
// and the nodes with a record.
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
mods, err := indexerView(s.ask, indexerModules)
mods, err := s.st.keys(modulesState)
if err != nil {
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
}
held := map[string]bool{}
for k := range mods {
if name, err := meshcatalogclient.ParseModulesKey(k); err == nil {
for _, k := range mods {
if name, err := delivery.ParseModulesKey(k); err == nil {
held[name] = true
}
}
@@ -151,12 +109,12 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
@@ -165,30 +123,42 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
}
// proposal reads one open proposal.
func (s *seatTools) proposal(id string) (mma.Proposal, error) {
if err := mmac.CheckProposalID(id); err != nil {
return mma.Proposal{}, err
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
if err := delivery.CheckProposalID(id); err != nil {
return delivery.PlacementProposal{}, err
}
raw, _, found, err := s.st.get(mma.StateProposals, id)
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
if err != nil {
return mma.Proposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
}
if !found {
return mma.Proposal{}, fmt.Errorf("no proposal %s is open: mesh-module-assigner.proposals lists those that are", id)
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
}
return plc.DecodeProposal(raw)
return delivery.DecodePlacementProposal(raw)
}
// write writes a proposal laid by assign.Propose, whose write the assigner's process takes as the event to lay it: it writes the
// candidates, each with what it adds (design 47 G A-317, A-316); the verb writes none, so no node-engine hears a candidate before
// its parts.
func (s *seatTools) write(p mma.Proposal) error {
p.Written = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
// takes as the event to compose and judge it.
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
p.Written = s.now().UTC()
for _, node := range p.Nodes() {
key, err := delivery.PlacementCandidateKey(p.ID, node)
if err != nil {
return err
}
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
return fmt.Errorf("the candidate %s is not written: %w", key, err)
}
}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
}
return nil
@@ -196,17 +166,17 @@ func (s *seatTools) write(p mma.Proposal) error {
// proposed is what propose and adopt answer.
type proposed struct {
Proposal mma.Proposal `json:"proposal"`
Follow string `json:"follow"`
Proposal delivery.PlacementProposal `json:"proposal"`
Follow string `json:"follow"`
}
func followProposal(id string) string {
return fmt.Sprintf("the nodes it touches answer it; mesh-module-assigner.proposals with id %q says where it stands, and the "+
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
"operator is asked to approve it once its verdict passes", id)
}
// changesOf reads propose's changes: a JSON array, given as one or as its text.
func changesOf(v any) ([]mma.ProposalChange, error) {
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
var raw []byte
switch c := v.(type) {
case string:
@@ -219,7 +189,7 @@ func changesOf(v any) ([]mma.ProposalChange, error) {
return nil, err
}
}
var out []mma.ProposalChange
var out []delivery.PlacementProposalChange
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
@@ -235,11 +205,11 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
return nil, err
}
for _, c := range changes {
if c.Op == mmac.OpSeed {
if c.Op == delivery.OpSeed {
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
current, revs, err := s.assignments()
current, revs, err := s.placements()
if err != nil {
return nil, err
}
@@ -251,24 +221,23 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
if err != nil {
return nil, err
}
enrollers := plc.EnrollersOf(current)
now := s.now()
p, _, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()})
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" {
findings = append(findings, refusal)
}
if len(findings) > 0 {
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p); err != nil {
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
func warrantOf(v any) (*mma.Warrant, error) {
func warrantOf(v any) (*delivery.AddNodeWarrant, error) {
var raw []byte
switch w := v.(type) {
case nil:
@@ -284,7 +253,7 @@ func warrantOf(v any) (*mma.Warrant, error) {
return nil, err
}
}
var out mma.Warrant
var out delivery.AddNodeWarrant
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
@@ -293,20 +262,18 @@ func warrantOf(v any) (*mma.Warrant, error) {
return &out, nil
}
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: the holder of the seat
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
// `person.<module>`), else the caller as proven, which the refusal names.
func moduleOfCaller(by string, enrollers []string) string {
for _, module := range enrollers {
if (stdio.Callers{module}).Allows(by) {
return module
}
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
// as proven, which the refusal names.
func moduleOfCaller(by string) string {
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
return delivery.EnrolmentModule
}
return by
}
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
// assignments-withdrawn carrying it whole.
// placement-withdrawn carrying it whole.
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
why := arg(args, "why")
if why == "" {
@@ -317,11 +284,11 @@ func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
return nil, err
}
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
raw, err := plc.EncodeProposal(p)
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
@@ -334,46 +301,19 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
if err != nil {
return nil, err
}
p.Again = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
p.Again = s.now().UTC()
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// doPlacement is the verb placement (design 47 G A-62, §704): the module's placements as the assignments rows hold them, one
// per branch a row names, with the nodes that run the module from it. A row naming no branch is one placement with the branch
// empty, as the row holds it.
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
module := arg(args, "module")
if module == "" {
return nil, errors.New("placement takes the module")
}
all, _, err := s.assignments()
if err != nil {
return nil, err
}
byBranch := map[string][]string{}
for node, a := range all {
for _, m := range a.Modules {
if m.Module == module {
byBranch[m.Branch] = append(byBranch[m.Branch], node)
}
}
}
out := mma.PlacementAnswer{Placements: []mma.ModulePlacement{}}
for _, branch := range slices.Sorted(maps.Keys(byBranch)) {
out.Placements = append(out.Placements, mma.ModulePlacement{Branch: branch, Nodes: slices.Sorted(slices.Values(byBranch[branch]))})
}
return out, nil
}
// assignmentsAnswer is what the verb assignments answers.
type assignmentsAnswer struct {
// placementAnswer is what the verb placement answers.
type placementAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Count int `json:"count"`
@@ -384,38 +324,38 @@ type assignmentsAnswer struct {
// historyEntry is one change of a key in the state's revisions.
type historyEntry struct {
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change plc.Change `json:"change"`
Deleted bool `json:"deleted,omitempty"`
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change delivery.PlacementChange `json:"change"`
Deleted bool `json:"deleted,omitempty"`
}
// doAssignments is the verb assignments (§10, §6): one node's key, one module's nodes, or every key; with history, the
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
// changes in order from the revisions the state keeps.
func (s *seatTools) doAssignments(args map[string]any) (any, error) {
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
if arg(args, "seats") == "true" {
return s.seats(arg(args, "node"))
}
all, revs, err := s.assignments()
all, revs, err := s.placements()
if err != nil {
return nil, err
}
node, module := arg(args, "node"), arg(args, "module")
out := assignmentsAnswer{Keys: map[string]keyed{}, Count: len(all), History: plc.AssignmentsHistory}
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
for n, p := range all {
if node != "" && n != node {
continue
}
if module != "" && !strings.Contains(" "+strings.Join(mmac.Assigned(p), " ")+" ", " "+module+" ") {
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
continue
}
out.Keys[n] = keyed{Assignments: p, Revision: revs[n]}
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
out.Nodes = append(out.Nodes, n)
}
sort.Strings(out.Nodes)
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
return nil, fmt.Errorf("%s has no assignments key", node)
return nil, fmt.Errorf("%s has no placement key", node)
}
if arg(args, "history") != "true" {
return out, nil
@@ -453,8 +393,8 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
out.Changes = append(out.Changes, e)
}
}
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its assignments-changed event on "+
"the seat's subject on the shared events stream, kept %d days", plc.AssignmentsHistory, plc.EventDays)
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
return out, nil
}
@@ -462,18 +402,18 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
Proposal delivery.PlacementProposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
func (s *seatTools) doProposals(args map[string]any) (any, error) {
ids, err := s.st.keys(mma.StateProposals)
ids, err := s.st.keys(delivery.PlacementProposalsState)
if err != nil {
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
}
cands, err := s.st.keys(mma.StateCandidates)
cands, err := s.st.keys(delivery.PlacementCandidatesState)
if err != nil {
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
}
@@ -481,9 +421,9 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
@@ -491,13 +431,13 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
h, err := nodeengineclient.DecodeHaveNode(raw)
h, err := delivery.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]nodeengine.CandidateAnswer{}
answers[id] = map[string]delivery.CandidateAnswer{}
}
answers[id][node] = a
}
@@ -514,11 +454,11 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
}
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
for _, k := range cands {
pid, node, err := plc.ParseCandidateKey(k)
pid, node, err := delivery.ParsePlacementCandidateKey(k)
if err != nil || pid != id {
continue
}
if _, rev, found, err := s.st.get(mma.StateCandidates, k); err == nil && found {
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
o.Candidates[node] = rev
}
}
@@ -533,43 +473,48 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
// only from a tool of that name), with the seat's own schemas.
func (s *seatTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
input := map[string]map[string]any{}
description := map[string]string{}
for _, v := range seat.Serves {
input[v.Name], description[v.Name] = v.Input, v.Description
}
run := map[string]func(map[string]any) (any, error){
mma.VerbAssignments: s.doAssignments,
mma.VerbProposals: s.doProposals,
mma.VerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
mma.VerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
mma.VerbAgain: s.doAgain,
mma.VerbPlacement: s.doPlacement,
catalogue.PlacementVerbPlacement: s.doPlacement,
catalogue.PlacementVerbProposals: s.doProposals,
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbAgain: s.doAgain,
}
var out []stdio.Tool
for _, v := range mma.Verbs() {
out = append(out, stdio.Tool{Name: mma.Seat + "." + v, Run: run[v]})
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
}
return out
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
keys, err := s.st.keys(mma.StateSeatHolders)
keys, err := s.st.keys(delivery.SeatHoldersState)
if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
}
sort.Strings(keys)
out := []mma.SeatHolders{}
out := []delivery.SeatHolders{}
for _, key := range keys {
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
}
if !found {
continue
}
var sh mma.SeatHolders
var sh delivery.SeatHolders
if json.Unmarshal(raw, &sh) != nil {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
continue
}
out = append(out, sh)
@@ -577,56 +522,76 @@ func (s *seatTools) seats(node string) (any, error) {
return map[string]any{"seats": out, "holds": len(keys)}, nil
}
// keyedRevision is one revision of a node's assignments key, the key it is of beside it.
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
type keyedRevision struct {
stdio.StateRevision
Key string
}
// nodeHistory is the revisions of a node's assignments, in the order the bus wrote them: its node key's `<node>.node`,
// whose every revision is one write of the node (research 072 row 37: the node key is a write's commit, so its history is
// the node's assignments history).
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
nodeKey, err := mmac.NodeKey(node)
legacy, err := delivery.LegacyPlacementKey(node)
if err != nil {
return nil, err
}
revisions, err := s.st.history(mma.StateAssignments, nodeKey)
nodeKey, err := delivery.PlacementNodeKey(node)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", nodeKey, err)
return nil, err
}
var out []keyedRevision
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: nodeKey})
for _, key := range []string{legacy, nodeKey} {
revisions, err := s.st.history(delivery.PlacementState, key)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
}
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: key})
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
return out, nil
}
// sketch is a revision of a node's assignments as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (plc.ChangeOf).
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (delivery.ChangeOf).
type sketch struct {
modules []mma.AssignmentsByNodeNodeValueModulesItem
modules []delivery.PlacementRef
nodeWide string
}
// sketchOf reads one revision of a node key; a value that does not read places nothing.
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
func sketchOf(key string, value []byte) sketch {
_, _, kind, err := mmac.ParseKey(key)
if err != nil || kind != mmac.NodeKind {
return sketch{}
}
n, err := mmac.DecodeNode(value)
_, _, kind, err := delivery.ParsePlacementKey(key)
if err != nil {
return sketch{}
}
raw, _ := json.Marshal([]any{n.Pins, n.Holds})
return sketch{modules: n.Modules, nodeWide: string(raw)}
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
raw, _ := json.Marshal([]any{pins, holds})
return string(raw)
}
if kind == delivery.PlacementNodeKind {
n, err := delivery.DecodePlacementNode(value)
if err != nil {
return sketch{}
}
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
}
p, err := delivery.DecodePlacement(value)
if err != nil {
return sketch{}
}
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
for _, m := range p.Modules {
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
}
return out
}
// changeFrom is what changed from before to s on node, as plc.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) plc.Change {
c := plc.Change{Node: node}
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
c := delivery.PlacementChange{Node: node}
was := map[string]string{}
for _, r := range before.modules {
was[r.Module] = r.Digest
@@ -5,8 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"os"
"reflect"
"slices"
@@ -14,26 +12,14 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatlist"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/provisions"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
)
func (s memStore) del(state, key string) error {
@@ -50,47 +36,30 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
}
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; assignments and
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := mma.Assignments{V: keys.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}, {Module: "mesh-enrolment"}},
Holds: []mma.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: "mesh-enrolment"}}}
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
st := memStore{}
for _, state := range []string{mma.StateAssignments, mma.StateProposals, mma.StateCandidates, haveState} {
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
}
// Issue 1966: the indexer's modules are asked of its first view, never read from its bus state (ADR 0596, ADR 0621).
indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {}}}
for _, m := range []string{"web", "shell", "mesh-enrolment"} {
indexer.views["mesh-delivery-indexer.modules"][m] = []byte("{}")
for _, m := range []string{"web", "shell"} {
st[modulesState].values[m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: n}})
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
var asked []string
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if raw, ok, err := indexer.answer(key, body); ok {
return raw, err
}
return json.RawMessage(`{"ok":true}`), nil
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
setAssignments(st[mma.StateAssignments], anchor)
// placement (design 47 G A-62): a module's placements as the rows hold them, one per branch, with the nodes that run it from there.
before := maps.Clone(st[mma.StateAssignments].values)
setAssignments(st[mma.StateAssignments], mma.Assignments{V: keys.Version, Node: "laptop", Modules: []mma.PlacedModule{{Module: "web", Branch: "feat/x"}}})
if got, err := s.doPlacement(map[string]any{"module": "web"}); err != nil ||
!reflect.DeepEqual(got, mma.PlacementAnswer{Placements: []mma.ModulePlacement{{Branch: "", Nodes: []string{"anchor"}}, {Branch: "feat/x", Nodes: []string{"laptop"}}}}) {
t.Errorf("placement of web = %+v, %v", got, err)
}
if got, err := s.doPlacement(map[string]any{"module": "ghost"}); err != nil || len(got.(mma.PlacementAnswer).Placements) != 0 {
t.Errorf("placement of a module placed nowhere = %+v, %v", got, err)
}
st[mma.StateAssignments].values = before
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
// withdraw and again mark the proposal; the holder ends or judges it.
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
@@ -104,18 +73,18 @@ func TestTheSeatsVerbs(t *testing.T) {
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
t.Fatal(err)
}
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again == nil || p.Again.IsZero() {
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
t.Errorf("marked = %+v, %v", p, err)
}
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
t.Error("an unknown proposal was withdrawn")
}
delete(st[mma.StateProposals].values, seed.ID) // the holder ended it
st[mma.StateCandidates].values = map[string][]byte{}
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
!strings.Contains(err.Error(), "ghost") || len(st[mma.StateProposals].values) != 0 {
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
t.Errorf("a refused proposal: %v", err)
}
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
@@ -127,46 +96,43 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Fatal(err)
}
move := got.(proposed).Proposal
if move.By != "operator@laptop" || move.Base["anchor"] != int64(len(st[mma.StateAssignments].values["anchor.node"])) || move.Base["laptop"] != 0 || len(st[mma.StateCandidates].values) != 0 {
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
t.Fatalf("propose = %+v", move)
}
// assignments and proposals read every state the seat keeps (§10); history names the stream for what is older.
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
reads := map[string]bool{}
pa, err := s.doAssignments(map[string]any{"module": "web", "history": "true"})
if a := pa.(assignmentsAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, "shared events stream") {
t.Errorf("assignments of web = %+v, %v", pa, err)
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
t.Errorf("placement of web = %+v, %v", pa, err)
} else {
reads[mma.StateAssignments], reads["events"] = true, true
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
}
if _, err := s.doAssignments(map[string]any{"node": "laptop"}); err == nil {
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
t.Error("a node with no key was answered")
}
ps, err := s.doProposals(map[string]any{"id": move.ID})
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 0 {
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
t.Errorf("proposals = %+v, %v", ps, err)
} else {
reads[mma.StateProposals], reads[mma.StateCandidates] = true, true
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
}
raw0, err := readManifest()
if err != nil {
t.Fatal(err)
}
for _, state := range raw0.State {
if slices.Contains([]string{mma.StateAssignments, mma.StateProposals, mma.StateCandidates}, state.Name) && !reads[state.Name] {
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
t.Errorf("no verb reads the seat's state %s", state.Name)
}
}
for _, stream := range raw0.Streams {
if strings.HasSuffix(stream.Name, "-findings") { // the part's own finding events, not a state of the seat
continue
}
if !reads["stream "+stream.Name] {
t.Errorf("no verb reads the seat's stream %s", stream.Name)
}
}
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand assignments back to.
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
t.Errorf("a hold: %v", err)
}
@@ -192,10 +158,11 @@ func TestTheSeatsVerbs(t *testing.T) {
if err := addNode("novox.mesh-enrolment", good); err != nil {
t.Errorf("an add-node from mesh-enrolment: %v", err)
}
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off assignments: from
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
// of a node running modules, or with another node's warrant, it is refused.
setAssignments(st[mma.StateAssignments], mma.Assignments{V: keys.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}})
st[delivery.PlacementState].values["placement.joiner"] = joiner
removeNode := func(node, warrant string) (any, error) {
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
@@ -210,88 +177,24 @@ func TestTheSeatsVerbs(t *testing.T) {
}
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
t.Errorf("a remove-node from mesh-enrolment: %v", err)
} else if rm := got.(proposed).Proposal; !slices.Equal(plc.Nodes(rm), []string{"joiner"}) || !slices.Equal(plc.RemovedNodes(rm), []string{"joiner"}) {
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
t.Errorf("the remove-node proposed %+v", rm)
}
// 1126/B8: the seat serves its six verbs; adopt and hand-back went with the controller (ADR 0590).
if tools := s.tools(); len(tools) != 6 || tools[2].Name != "mesh-module-assigner.propose" || tools[2].Run == nil {
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
// Issue 1966: propose checks its modules against the indexer's first view of modules, every page of it, asked again at
// each call; a first view the indexer refuses refuses the proposal, naming the tool, and writes nothing.
for _, c := range []struct {
name, module, refused, want string
}{
{"a module on the last page", "web", "", ""},
{"a module the indexer does not hold", "ghost", "", "ghost"},
{"the first view refused", "web", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.modules does not answer"},
} {
t.Run("1966: "+c.name, func(t *testing.T) {
indexer.refused, indexer.asks = c.refused, nil
maps.DeleteFunc(st[mma.StateAssignments].values, func(k string, _ []byte) bool { return strings.HasPrefix(k, "joiner.") }) // the remove-node's joiner
st[mma.StateProposals].values, st[mma.StateCandidates].values = map[string][]byte{}, map[string][]byte{}
_, err := s.doPropose(map[string]any{"why": "x", "changes": []any{map[string]any{"op": "place", "node": "laptop", "module": c.module}}}, "op")
switch {
case c.want == "" && err != nil:
t.Errorf("propose: %v", err)
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want) || len(st[mma.StateProposals].values) != 0):
t.Errorf("propose: %v, want a refusal saying %q and nothing written", err, c.want)
}
if pages := indexer.asks["mesh-delivery-indexer.modules"]; c.refused == "" && pages != 3 {
t.Errorf("the first view of modules was asked %d pages, want its 3", pages)
}
})
}
indexer.refused = ""
// Issue 1966: the replicated mesh-dns-resolver held on two nodes is proposed through the verb as the bundle serves it,
// over the seat list it reads (ADR 0636 §1): unread, the check says it waits for the list and writes nothing; read, the
// proposal is laid.
for _, c := range []struct {
name string
silent bool
want string
}{
{"a replicated seat on two nodes waits for an unread seat list", true, "the seat list is not read yet, so whether the seat mesh-dns-resolver may be held on anchor and laptop waits for it"},
{"a replicated seat on two nodes, the seat list read", false, ""},
} {
t.Run("1966: "+c.name, func(t *testing.T) {
st[mma.StateProposals].values, st[mma.StateCandidates].values = map[string][]byte{}, map[string][]byte{}
s.seatList = gateOver(t, c.silent)
_, err := s.doPropose(map[string]any{"why": "a second resolver", "changes": []any{
map[string]any{"op": "hold", "node": "anchor", "module": "web", "claim": "mesh-dns-resolver", "scope": "mesh"},
map[string]any{"op": "place", "node": "laptop", "module": "web"},
map[string]any{"op": "hold", "node": "laptop", "module": "web", "claim": "mesh-dns-resolver", "scope": "mesh"}}}, "op")
switch {
case c.want == "" && err != nil:
t.Errorf("propose: %v", err)
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want) || strings.Contains(err.Error(), "held once") || len(st[mma.StateProposals].values) != 0):
t.Errorf("propose: %v, want a refusal saying %q, never held once, and nothing written", err, c.want)
}
})
}
s.seatList = nil
t.Run("1966: the seat list's again tool answers an operator's session through the mesh MCP server", func(t *testing.T) {
// Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. The list names
// the seats, resolved from the assigner's own seat-holders (A-295, A-298): a module holding neither is refused.
held := map[string][]byte{"node-tool-runner.shanks": []byte(`{"holders":[{"module":"node-tools","node":"shanks"}]}`),
"mesh-healers": []byte(`{"holders":[{"module":"mesh-delivery-watchman","node":"ace"}]}`)}
holders := seatlist.HoldersIn(func() (map[string][]byte, error) { return held, nil })
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-delivery-watchman": false,
"ace.mesh-healers": true, "ace.messenger": true} {
if err := seatlist.RefusalWith("assigner_seat_list_again", seatListAgainCallers, caller, holders); (err != nil) != refused {
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
}
}
})
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
putAssignments(w, mma.Assignments{V: keys.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
pkey, _ := delivery.LegacyPlacementKey("ace")
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
hn, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
hk, _ := nodeengineclient.HaveNodeKey(node)
w.putEntry(buckets.Have, hk, hn, 1, time.Now())
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
hk, _ := delivery.HaveNodeKey(node)
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("ace: %+v %v", d, ok)
@@ -301,217 +204,142 @@ func TestTheSeatsVerbs(t *testing.T) {
}
findings := map[string][]byte{}
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
Put: putInto(findings),
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
p.unplacedFindings()
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
putAssignments(w, mma.Assignments{V: keys.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
pkey2, _ := delivery.LegacyPlacementKey("g14")
w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now())
p.unplacedFindings()
if len(findings) != 0 {
t.Errorf("placed again: findings %v", findings)
}
})
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from assignments, only the keys a change moves, and answered by assignments seats", func(t *testing.T) {
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
writes := 0
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
// Issue 1966: nothing is written while the seat list is unread (ADR 0636 §1), and every seat the list names is no
// seat outside it once read.
p := &passer{w: newWorld(), logf: t.Logf, kvs: kvSet{plc.SeatHoldersBucket(): kv}, seats: gateOver(t, true)}
place := func(node string, holds ...mma.Holding) {
putAssignments(p.w, mma.Assignments{V: keys.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}}
place := func(node string, holds ...delivery.Holding) {
pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
if err != nil {
t.Fatal(err)
}
k, _ := delivery.LegacyPlacementKey(node)
p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now())
}
place("ace", mma.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
place("g14", mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
ctx := context.Background()
if p.seatHolders(ctx); writes != 0 {
t.Fatalf("seat-holders was written from an unread seat list: %v", kv.values)
}
p.seats = gateOver(t, false)
p.seatHolders(ctx)
var broker mma.SeatHolders
if err := json.Unmarshal(kv.values["mesh-broker"], &broker); err != nil || broker.Outside {
t.Errorf("mesh-broker's holders %+v, %v: a seat the list names is no seat outside it", broker, err)
}
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
t.Fatalf("%d writes: %v", writes, kv.values)
}
p.seatHolders(ctx)
if writes != 3 {
t.Errorf("an unchanged assignments wrote %d keys", writes-3)
t.Errorf("an unchanged placement wrote %d keys", writes-3)
}
place("g14") // g14 holds nothing now
p.seatHolders(ctx)
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
}
st := memStore{mma.StateSeatHolders: &memKV{values: kv.values}}
st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}}
out, err := newSeatTools(st, nil).seats("ace")
if err != nil || len(out.(map[string]any)["seats"].([]mma.SeatHolders)) != 2 {
if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 {
t.Errorf("seats on ace = %+v, %v", out, err)
}
})
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
t.Run("research 072 row 37: the form before is converted once, a node is written as its keys, and history reads both", func(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
p := &passer{w: newWorld(), kvs: kvSet{plc.AssignmentsBucket(): kv}, now: time.Now, logf: t.Logf}
first := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}
if _, err := p.writeNode(ctx, "anchor", &first); err != nil || strings.Join(kv.log, " ") != "create anchor.web create anchor.node" {
t.Fatalf("the first write = %v, %v", kv.log, err)
var logs []string
p := &passer{w: newWorld(), kvs: buckets{delivery.PlacementBucket: kv}, now: time.Now,
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
put := func(key string, value []byte, rev uint64) {
p.w.putEntry(delivery.PlacementBucket, key, value, rev, time.Now())
}
raw := kv.values["anchor.node"]
// A write: the module keys, the node key (its commit), then the module key no longer placed.
after := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell", Ports: map[string]int64{"22": 2222}}}}
was := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
raw, _ := delivery.EncodePlacement(was)
put("placement.anchor", raw, kv.write("placement.anchor", raw))
p.convertPlacement(ctx)
if strings.Join(kv.log, " ") != "create anchor.web create anchor.node" || !p.convertedAll ||
!strings.Contains(strings.Join(logs, "\n"), "anchor converted from placement.anchor") {
t.Fatalf("the conversion wrote %v, said %v", kv.log, logs)
}
p.convertPlacement(ctx)
if len(kv.log) != 2 {
t.Errorf("a converted node is converted again: %v", kv.log)
}
// A write: the module keys, the node key (its commit), the mirror, then the module key no longer placed.
after := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2222}}}}
kv.log = nil
rev, err := p.writeNode(ctx, "anchor", &after)
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node delete anchor.web" || rev != kv.revs["anchor.node"] {
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node update placement.anchor delete anchor.web" || rev != kv.revs["anchor.node"] {
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
}
if got, _ := p.seatAssignments(); !mmac.SameAssignments(got["anchor"], after) {
if got, _ := p.seatPlacements(); !delivery.SamePlacement(got["anchor"], after) {
t.Errorf("the write is not read back at once: %+v", got)
}
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
kv.refuse, kv.log = "anchor.node", nil
moved := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell", Ports: map[string]int64{"22": 2223}}}}
moved := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2223}}}}
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
t.Fatalf("a refused commit = %v, %v", kv.log, err)
}
fresh, err := mmac.AssignmentsOf(kv.values)
if err != nil || !mmac.SameAssignments(fresh["anchor"], after) {
fresh, err := delivery.PlacementsOf(kv.values)
if err != nil || !delivery.SamePlacement(fresh["anchor"], after) {
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
}
kv.refuse = ""
// The verb's history reads the node key.
st := memStore{mma.StateAssignments: &memKV{values: kv.values}}
// The verb's history reads the form before, then the node key.
st := memStore{delivery.PlacementState: &memKV{values: kv.values}}
hs := newSeatTools(st, nil)
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
"anchor.node": {{Key: "anchor.node", Value: raw, Revision: 1, Op: "put"}, {Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
"placement.anchor": {{Key: "placement.anchor", Value: raw, Revision: 1, Op: "put"}},
"anchor.node": {{Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
}}
pa, err := hs.doAssignments(map[string]any{"node": "anchor", "history": "true"})
a, _ := pa.(assignmentsAnswer)
pa, err := hs.doPlacement(map[string]any{"node": "anchor", "history": "true"})
a, _ := pa.(placementAnswer)
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
t.Errorf("history = %+v, %v", pa, err)
}
// No assignments: the node key, then the module keys.
// No placement: the mirror first, then the node key, then the module keys.
kv.log = nil
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete anchor.node delete anchor.shell" {
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete placement.anchor delete anchor.node delete anchor.shell" {
t.Errorf("the delete = %v, %v", kv.log, err)
}
})
t.Run("ADR 0641: the retired state is copied into assignments, key by key, with no gap and nothing deleted", func(t *testing.T) {
ctx := context.Background()
web, _ := mmac.AssignmentsValues(mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}})
shell, _ := mmac.AssignmentsValues(mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell"}}})
oldWeb := func() map[string][]byte {
return map[string][]byte{"anchor.node": web["anchor.node"], "anchor.web": web["anchor.web"]}
}
withLegacy := oldWeb()
withLegacy["placement.anchor"] = []byte(`{"whole":1}`)
withOdd := oldWeb()
withOdd["a.b.c"] = []byte("x")
for _, c := range []struct {
name string
old map[string][]byte // the retired state
marked bool // the copy ended at an earlier start
present map[string][]byte // what assignments holds at the start
wantLog string // the writes into assignments, in order
wantValues map[string][]byte // assignments afterwards
journal int // lines said
said copiedSaid // the event; none when zero
}{
{"an empty new state takes every key, the node key last, and the old form is not copied", withLegacy, false, nil,
"create anchor.web create anchor.node", web, 1, copiedSaid{Copied: 2, Dropped: 1}},
{"a copy cut short by a crash is completed at the next start, and the marker is written", oldWeb(), false,
map[string][]byte{"anchor.web": web["anchor.web"]},
"create anchor.node", web, 1, copiedSaid{Copied: 1, Present: 1}},
{"a start after the marker copies nothing, even with a key missing", oldWeb(), true,
map[string][]byte{"anchor.node": shell["anchor.node"], "anchor.shell": shell["anchor.shell"]},
"", shell, 0, copiedSaid{}},
{"a key that fits no form is left and counted", withOdd, false, nil,
"create anchor.web create anchor.node", web, 1, copiedSaid{Copied: 2, Skipped: 1}},
{"an empty old state copies nothing and says nothing", nil, false, nil, "", nil, 0, copiedSaid{}},
} {
t.Run(c.name, func(t *testing.T) {
next := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
for k, v := range c.present {
next.write(k, v)
}
old := &retiredKV{values: map[string][]byte{}}
maps.Copy(old.values, c.old)
copies := &copiesKV{values: map[string][]byte{}}
if c.marked {
copies.values[copiedMarker] = []byte(`{}`)
}
var logs []string
bus := &emitted{}
p := &passer{w: newWorld(), kvs: kvSet{retiredBucket(): old, plc.AssignmentsBucket(): next, copiesBucket(): copies}, now: time.Now, seat: &seatActs{bus: bus},
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
if !p.copyRetired(ctx) || !p.copied {
t.Fatalf("the copy did not end: %v", logs)
}
if got := strings.Join(next.log, " "); got != c.wantLog {
t.Errorf("writes = %q, want %q", got, c.wantLog)
}
if c.wantValues != nil && !maps.EqualFunc(next.values, c.wantValues, slices.Equal[[]byte]) {
t.Errorf("assignments = %v, want %v", next.values, c.wantValues)
}
if _, ok := copies.values[copiedMarker]; !ok {
t.Errorf("the end of the copy is not marked in %s: %v", copiesBucket(), copies.values)
}
if !maps.EqualFunc(old.values, c.old, slices.Equal[[]byte]) || old.deletes != 0 {
t.Errorf("the retired state changed: %v, %d deletes; the readers delivered before still read it", old.values, old.deletes)
}
if len(logs) != c.journal {
t.Errorf("said %d lines %v, want %d", len(logs), logs, c.journal)
}
if c.said == (copiedSaid{}) {
if len(bus.events) != 0 {
t.Errorf("no copy was made, and said %v", bus.events)
}
} else if len(bus.events) != 1 || bus.events[0] != c.said {
t.Errorf("said %v, want the event %+v once", bus.events, c.said)
}
// A restart finds the marker: nothing is written, and no second event is said.
before := len(next.log)
p.copied = false
if !p.copyRetired(ctx) || len(next.log) != before || len(bus.events) > 1 {
t.Errorf("a second copy wrote %v, said %v", next.log[before:], bus.events)
}
if got := len(p.w.read(plc.AssignmentsBucket())); got != c.said.Copied {
t.Errorf("%d copied keys are in the world, want %d", got, c.said.Copied)
}
})
}
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{
Put: putInto(findings),
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"})
p.w.putEntry(buckets.Provisions, "ace.photos.db", failed, 1, time.Now())
p.kvs[buckets.Provisions] = &memKV{}
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"})
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now())
p.kvs[provisions.ProvisionsBucket] = &memKV{}
of := p.provisionsOf()
if got := of("photos")("ace"); got["ace.photos.db"].State != meshprovisionerclient.Failed {
if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed {
t.Fatalf("provisions = %+v", got)
}
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Provided})
p.w.putEntry(buckets.Provisions, "ace.photos.db", provided, 2, time.Now())
provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided})
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now())
p.provisionsOf()
if len(findings) != 0 {
t.Errorf("findings after provided = %v", findings)
}
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
ctx := context.Background()
p.kvs[plc.PulsesBucket()] = &memKV{values: map[string][]byte{}}
p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
gone := map[string]bool{"ace.photos.s3": true}
p.provisionsMissing(ctx, gone)
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
@@ -521,7 +349,7 @@ func TestTheSeatsVerbs(t *testing.T) {
if len(findings) != 0 {
t.Fatal("said before its bound")
}
p.w.put(plc.PulsesBucket(), "provision-missing.ace.photos.s3", nil) // the server expired it
p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
p.provisionsMissing(ctx, gone)
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
t.Fatalf("past its bound: findings %v", findings)
@@ -537,44 +365,37 @@ func TestTheSeatsVerbs(t *testing.T) {
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
func wantsVerbs(t *testing.T) {
fpOld, fpNew := meshcatalogclient.SourcePrefix2+strings.Repeat("1", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("2", 64)
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{policiesState, haveState, putsState, "findings", buckets.ComposedRead} {
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
st[state] = &memKV{values: map[string][]byte{}}
}
// Issue 1966: the indexer's modules and manifests are asked of its first views, never read from its bus state.
indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {}, "mesh-delivery-indexer.manifests": {}}}
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
// a fingerprint, and laptop's web refused there.
partNew, _ := catalogkey.KeyPart(fpNew)
webKey, _ := nodeengineclient.ComposedKey("anchor", "web", partNew)
refusedKey, _ := nodeengineclient.ComposedKey("laptop", "web", partNew)
st[buckets.ComposedRead].values["anchor.node"], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor"})
st[buckets.ComposedRead].values[webKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor", Module: "web", FP: fpNew})
st[buckets.ComposedRead].values[refusedKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := meshcatalogclient.ManifestDigest([]byte(raw))
indexer.views["mesh-delivery-indexer.manifests"][digest], _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: name, Manifest: raw})
indexer.views["mesh-delivery-indexer.modules"][name], _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{V: keys.Version, Fp2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
digest := delivery.ManifestDigest([]byte(raw))
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
}
st[haveState].values["anchor.node"], _ = nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: "sha256:" + strings.Repeat("c", 64)})
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
var asked []string
var said []string
ask := func(key string, body any) (json.RawMessage, error) {
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if raw, ok, err := indexer.answer(key, body); ok {
return raw, err
}
return json.RawMessage(`{"ok":true}`), nil
}
w := newWantsTools(st, ask, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
}, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
policyOf := func(module string) policyRow {
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
if err != nil {
@@ -603,7 +424,7 @@ func wantsVerbs(t *testing.T) {
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
// large states (the modules, their manifests, the composed record).
watches := map[string]int{}
byWatch := newWantsTools(st, ask, nil)
byWatch := newWantsTools(st, nil, nil)
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
watches[state]++
for k, v := range st[state].values {
@@ -624,13 +445,10 @@ func wantsVerbs(t *testing.T) {
t.Errorf("compare by watch = %+v", got)
}
}
for _, state := range []string{"mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests"} {
if watches[state] != 0 || watches[state+" by key"] != 0 || indexer.asks[state] == 0 {
t.Errorf("the indexer's %s was watched %d times and read by key %d, asked of its first view %d times: want asked alone",
state, watches[state], watches[state+" by key"], indexer.asks[state])
}
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
t.Errorf("the manifests were read by key %d times, want each once", n)
}
for _, state := range []string{haveState, assign.FindingsState} {
for _, state := range []string{modulesState, haveState, health.FindingsState} {
if watches[state] != 1 {
t.Errorf("%s was watched %d times, want once", state, watches[state])
}
@@ -675,62 +493,9 @@ func wantsVerbs(t *testing.T) {
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
t.Errorf("bus = %+v, asked %v", b, asked)
}
// W5.5 (design 47 lines 1365-1366): every verb of the seat is served, revision, entry and difference reading only the
// kept revisions.
objects := memObjects{}
w.revisions = objects
tools := w.tools()
run := map[string]func(map[string]any) (any, error){}
for _, tool := range tools {
if tool.Run == nil {
t.Errorf("the seat's verb %s is not served", tool.Name)
}
run[tool.Name] = tool.Run
}
// the verb input is answered by the program in chunks (inputs.go), not as a tool
if len(tools) != 5 || tools[0].Name != "mesh-wants.upgrade" {
if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
if _, err := run["mesh-wants.revision"](map[string]any{}); !errors.Is(err, assigngraph.ErrNoRevision) {
t.Errorf("revision with none kept: %v", err)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{V: 1, Kind: "trunk", Placements: "sha256:p",
Catalogs: []meshwants.CatalogOf{{Branch: "main", Revision: 3}}, Entries: []meshwants.NodeEntryRef{{Node: "anchor", Module: "web", Digest: "sha256:e"}}},
Entries: map[string]meshwants.AssignmentEntry{"sha256:e": {Node: "anchor", Module: "web", Branch: "main", Fp: fpNew, Catalog: "sha256:c"}}}
if _, _, err := assigngraph.Keep(context.Background(), objects, c, "catalog-revision", time.Now()); err != nil {
t.Fatal(err)
}
r, err := run["mesh-wants.revision"](map[string]any{"revision": 1, "caller": "seat:mesh-grants"})
if err != nil || r.(meshwants.RevisionAnswer).Revision.Catalogs[0].Revision != 3 {
t.Errorf("revision = %+v, %v", r, err)
}
if e, err := run["mesh-wants.entry"](map[string]any{"node": "anchor", "module": "web"}); err != nil || e.(meshwants.EntryAnswer).Entry.Fp != fpNew {
t.Errorf("entry = %+v, %v", e, err)
}
if d, err := run["mesh-wants.difference"](map[string]any{"from": 1, "to": 1}); err != nil || len(d.(meshwants.Difference).Changes) != 0 {
t.Errorf("difference = %+v, %v", d, err)
}
}
// memObjects is an object bucket in memory.
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := m[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (m memObjects) Names(context.Context) ([]string, error) {
out := []string{}
for n := range m {
out = append(out, n)
}
return out, nil
}
// readManifest is the module's manifest, read as the controller reads it.
@@ -771,11 +536,8 @@ func (k *revKV) write(key string, value []byte) uint64 {
}
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
if _, ok := k.values[key]; ok {
return 0, jetstream.ErrKeyExists
}
if key == k.refuse {
return 0, errors.New("refused")
if _, ok := k.values[key]; ok || key == k.refuse {
return 0, errors.New("key exists")
}
k.log = append(k.log, "create "+key)
return k.write(key, value), nil
@@ -808,138 +570,3 @@ type historyStore struct {
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
return s.revisions[key], nil
}
// putAssignments takes a node's assignments into the world as the keys of the state: a key per module, then the node key.
func putAssignments(w *world, a mma.Assignments) {
values, err := mmac.AssignmentsValues(a)
if err != nil {
panic(err)
}
for _, k := range slices.Sorted(maps.Keys(values)) {
w.putEntry(plc.AssignmentsBucket(), k, values[k], 1, time.Now())
}
}
// setAssignments holds a node's assignments in a state as its keys: a key per module, and the node key.
func setAssignments(kv *memKV, a mma.Assignments) {
values, err := mmac.AssignmentsValues(a)
if err != nil {
panic(err)
}
maps.Copy(kv.values, values)
}
// retiredKV is the retired state: its keys, its values, and deletes of them.
type retiredKV struct {
jetstream.KeyValue
values map[string][]byte
deletes int
}
func (k *retiredKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
if len(k.values) == 0 {
return nil, jetstream.ErrNoKeysFound
}
return slices.Sorted(maps.Keys(k.values)), nil
}
func (k *retiredKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := k.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return retiredEntry{key: key, value: v}, nil
}
func (k *retiredKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
k.deletes++
delete(k.values, key)
return nil
}
type retiredEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e retiredEntry) Value() []byte { return e.value }
func (e retiredEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
// emitted is a bus that keeps the module events it was asked to say.
type emitted struct {
caller
events []copiedSaid
}
func (e *emitted) Emit(eventType string, body any) error {
if eventType == CopiedEvent {
e.events = append(e.events, body.(copiedSaid))
}
return nil
}
func (k *revKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
if len(k.values) == 0 {
return nil, jetstream.ErrNoKeysFound
}
return slices.Sorted(maps.Keys(k.values)), nil
}
// copiesKV is the state copies: the marker of the end of a copy.
type copiesKV struct {
jetstream.KeyValue
values map[string][]byte
}
func (k *copiesKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := k.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return retiredEntry{key: key, value: v}, nil
}
func (k *copiesKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
k.values[key] = value
return uint64(len(k.values)), nil
}
// fakeIndexer answers the indexer's first-view tools as its runtime does (views.FirstViewTool): each tool its state's values
// in key order, one key a page, each page continued after the page before's next; the tool named refused answers its
// refusal. asks counts the pages asked by tool.
type fakeIndexer struct {
views map[string]map[string][]byte
refused string
asks map[string]int
}
// answer is the answer to a call of key, and whether key is one of the indexer's first views at all.
func (f *fakeIndexer) answer(key string, body any) (json.RawMessage, bool, error) {
values, ok := f.views[key]
if !ok {
return nil, false, nil
}
if f.asks == nil {
f.asks = map[string]int{}
}
f.asks[key]++
if key == f.refused {
return nil, true, fmt.Errorf("%s is refused to \"novox.mesh-delivery-assigner\"", key)
}
after, _ := body.(map[string]any)["after"].(string)
a := views.Answer{V: views.Version, Owner: buckets.IndexerModule, State: strings.TrimPrefix(key, buckets.IndexerModule+"."), Revision: 7,
Values: map[string]json.RawMessage{}}
for _, k := range slices.Sorted(maps.Keys(values)) {
if k <= after {
continue
}
if len(a.Values) == 1 {
a.Next = slices.Collect(maps.Keys(a.Values))[0]
break
}
a.Values[k] = values[k]
}
raw, err := json.Marshal(a)
return raw, true, err
}
@@ -1,16 +0,0 @@
package main
import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// (plc.CheckAssignments), and seat-holders is not written.
// A call through the mesh MCP server is proved only as its machine's tool runner (issue 627), so the holders of the seat
// node-tool-runner stand for an operator's session until a session is proved (issue 1966). The list names seats, never a
// module, resolved from the assigner's own seat-holders (design 47 G A-210, A-295, A-298).
var seatListAgainCallers = []string{stdio.AnyPerson, stdio.CallerSeatPrefix + "node-tool-runner", stdio.CallerSeatPrefix + "mesh-healers"}
@@ -1,114 +0,0 @@
package main
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
)
// testDefinitions are seatlisttest's, parsed once.
var testDefinitions = sync.OnceValue(func() []seatdef.Definition { return seatlisttest.Definitions() })
// standInCatalog is the seat mesh-catalog as a test's reader asks it (ADR 0636 §4): every definition of seatlisttest, or,
// silent, no answer at all.
type standInCatalog struct{ silent bool }
func (c standInCatalog) Ask(key string, body any) (json.RawMessage, error) {
if c.silent || key != seatlist.Key() {
return nil, errors.New("no responders")
}
defs := testDefinitions()
seat, _ := body.(map[string]any)["seat"].(string)
if seat == "" {
l := seatlist.Listing{Revision: 1}
for _, d := range defs {
l.Seats = append(l.Seats, seatlist.Entry{Seat: d.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + d.Seat})
}
return json.Marshal(l)
}
for _, d := range defs {
if d.Seat == seat {
raw, err := json.Marshal(d)
if err != nil {
return nil, err
}
return json.Marshal(seatlist.Answer{Seat: seat, Found: true, Revision: 1,
Entry: &seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "d-" + seat, Definition: raw}})
}
}
return json.Marshal(seatlist.Answer{Seat: seat, Revision: 1})
}
func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
// gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list
// (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere.
func gateOver(t *testing.T, silent bool) *seatlist.Part {
t.Helper()
if silent {
// A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test
// that waits for it to stay unread keeps short.
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
}
g := seatlist.NewPart("assigner", seatListAgainCallers)
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
if r == nil {
t.Fatal("the reader did not start")
}
<-r.Ready()
if g.Heard() == silent {
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health())
}
return g
}
// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the
// assigner's code finds no start of a reader and no tools of its own.
func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} {
if strings.Contains(string(raw), own) {
t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own)
}
}
}
}
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
if _, ok := seatlist.Definition(seatlist.NewPart("assigner", seatListAgainCallers).Reader, "mesh-wants")(); ok {
t.Error("a definition is answered before the seat list's reader started")
}
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
t.Error("a definition is answered while the seat mesh-catalog has not answered the list")
}
read := gateOver(t, false)
if d, ok := seatlist.Definition(read.Reader, "mesh-wants")(); !ok || d.Seat != "mesh-wants" {
t.Errorf("the read list's mesh-wants is answered as %v, %v", d.Seat, ok)
}
if _, ok := seatlist.Definition(read.Reader, "no-such-seat")(); ok {
t.Error("a seat the list does not hold is answered a definition")
}
}
@@ -1,15 +1,9 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/puts"
"os"
"regexp"
"slices"
@@ -18,27 +12,15 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
// (seat_tools.go, ADR 0567 §10).
// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again.
const engineReadsTool = "assigner_engine_reads"
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
@@ -54,9 +36,7 @@ type store interface {
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
type putTools struct {
st store
// ask asks a module's tool (stdio.Ask, or a test's): the indexer's first view of modules for newest.
ask func(key string, body any) (json.RawMessage, error)
st store
now func() time.Time
mu sync.Mutex
// heard is every put the watch handed, by key; changed is closed and replaced at each.
@@ -64,8 +44,8 @@ type putTools struct {
changed chan struct{}
}
func newPutTools(st store, ask func(string, any) (json.RawMessage, error)) *putTools {
return &putTools{st: st, ask: ask, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
func newPutTools(st store) *putTools {
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
}
// hear takes a put's change in from the watch.
@@ -77,66 +57,72 @@ func (s *putTools) hear(key string, value []byte) {
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole.
var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181:
// a put names the whole fingerprint, the builder's builds being no bucket to list).
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
case v == "newest":
// Asked of the indexer's first view of modules, never read from its bus state (ADR 0596, ADR 0621; issue 1966).
mods, err := indexerView(s.ask, indexerModules)
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
if err != nil {
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
raw, found := mods[module]
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := meshcatalogclient.DecodeModule(raw)
m, err := delivery.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.Fp2 == "" {
if m.FP2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.Fp2, nil
return m.FP2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version)
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
puts.Put
delivery.Put
Follow string `json:"follow,omitempty"`
// Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the
// node's want until it ends or is released.
Older string `json:"older-than-newest,omitempty"`
// Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is
// written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*).
Since time.Time `json:"since"`
Bound string `json:"bound,omitempty"`
}
// answerOf is what assigner_put and assigner_puts answer of the put p of module on node.
func answerOf(node, module string, p puts.Put) putAnswer {
a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()}
if putWaits(p) {
a.Bound = putWaitBound.String()
}
return a
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -160,17 +146,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before.FP == fp && before.Hold == hold {
a := answerOf(node, module, before)
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := puts.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, puts.Asked, words)
data, err := puts.Encode(p)
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -178,21 +162,19 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
a := answerOf(node, module, s.await(key, p))
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p puts.Put) puts.Put {
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := puts.Decode(raw); err == nil && now.At.Equal(p.At) {
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != puts.Asked && now.Step != puts.Build && now.Step != puts.Compose {
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
return p // it waits on a person or the node, or ended
}
}
@@ -212,17 +194,8 @@ func followPut(node, module string) string {
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if arg(args, "how") != "" {
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
}
if step := arg(args, "step"); step != "" {
if step != meshwants.WaitStepDelivery {
return nil, fmt.Errorf("step is %s, the one wait released by its step, not %q", meshwants.WaitStepDelivery, step)
}
return s.doReleaseDelivery(args, by) // a dependent's delivery wait (delivered.go, design 47 G §376, A-246)
}
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -236,15 +209,15 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if puts.Ended(p.Step) {
if delivery.PutEnded(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), puts.Released, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := puts.Encode(p)
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -252,7 +225,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return answerOf(node, module, p), nil
return putAnswer{Node: node, Module: module, Put: p}, nil
}
// doPuts is the tool assigner_puts.
@@ -263,10 +236,8 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
}
sort.Strings(keys)
out := []putAnswer{}
newest := map[string]string{} // the indexer's newest fingerprint by module, asked once and only for a put that stands
asked := false
for _, key := range keys {
node, module, err := puts.ParseKey(key)
node, module, err := delivery.ParsePutKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
@@ -274,27 +245,11 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
continue
}
a := answerOf(node, module, p)
if !puts.Ended(p.Step) {
if !asked {
asked = true
if mods, err := indexerView(s.ask, indexerModules); err == nil {
for name, raw := range mods {
if m, err := meshcatalogclient.DecodeModule(raw); err == nil {
newest[name] = m.Fp2
}
}
}
}
if n := newest[module]; n != "" && n != p.FP {
a.Older = n
}
}
out = append(out, a)
out = append(out, putAnswer{Node: node, Module: module, Put: p})
}
return map[string]any{"puts": out}, nil
}
@@ -351,14 +306,9 @@ func (s *putTools) tools() []stdio.Tool {
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
"failed (the failing check is the delivery's), rejudge (judge it again now), or now (the entry waiting for an acting part's new version moves with the running one); written in the hand-act log first",
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
[]string{"module", "why"}),
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
[]string{"node", "module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
s.waitsTool(),
s.deliveredAgainTool(),
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
@@ -386,8 +336,7 @@ func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(st
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
func serveTools() error {
plc.SetModule(os.Getenv(stdio.ServedAs)) // the module the runtime serves these tools as (A-296)
s := newPutTools(runtimeStore{}, stdio.Ask)
s := newPutTools(runtimeStore{})
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
@@ -402,38 +351,19 @@ func serveTools() error {
time.Sleep(wait)
}
}()
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
gate := seatlist.NewPart("assigner", seatListAgainCallers)
go func() {
<-stdio.Served()
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
}()
seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask)
seatVerbs.seatList = gate
tools := append(s.tools(), seatVerbs.tools()...)
tools = append(tools, gate.Tools()...)
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
return stdio.SeatPublish(wantsEventSubject(e), b, id)
})
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
tools = append(tools, wants.tools()...)
tools = append(tools, inputsRead(&inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}))
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
tools = append(tools, writtenTools(runtimeStore{})...)
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
return stdio.Serve("", tools)
}
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := catalogkey.KeyPart(fp)
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
@@ -1,14 +1,13 @@
package main
import (
"context"
"slices"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
@@ -18,8 +17,21 @@ import (
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
// manifests, as a provider is counted (ADR 0603 §5).
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
// moduleGraph is the module graph as the indexer last wrote it; a key that does not decode is left out, so its module
// waits on no verb.
func (p *passer) moduleGraph() graph.Graph {
g := graph.Graph{}
for key, raw := range p.w.read(graph.Bucket) {
module, err := graph.ParseKey(key)
if err != nil {
continue
}
if n, err := graph.Decode(raw); err == nil {
g[module] = n
}
}
return g
}
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
@@ -46,11 +58,11 @@ func keptInvokes(g graph.Graph, module string) []string {
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
// a verb.
func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool {
if len(g) == 0 {
return nil
}
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
@@ -59,22 +71,20 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
// that build serves what it offers (A-352); a consumer needing the newer build's forms waits by its kind-4 edge instead. A
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
// holds a consumer for good.
func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
offered := map[string]bool{}
for node, haves := range n.Haves {
if !n.Online[node] {
continue
}
for module, h := range haves {
if h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseWaitingYes || h.Health != nodeengineclient.HealthOK {
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
continue
}
m, ok := p.manifestOf(module, h.Fp)
m, ok := p.manifestOf(ctx, get, module, h.FP)
if !ok {
m = trunk[module]
}
@@ -87,22 +97,30 @@ func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) f
}
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
func (p *passer) servesOf(module, fp string) ([]string, bool) {
m, ok := p.manifestOf(module, fp)
func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) {
m, ok := p.manifestOf(ctx, get, module, fp)
if !ok {
return nil, false
}
return graph.ServesOf(m), true
}
// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests
// hold), and whether it could be read.
func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
e, ok := p.w.ends.endOf(module, fp)
if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" {
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
// and whether it could be read.
func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return catalogue.Manifest{}, false
}
got := p.w.manifest(e.Manifest)
raw := p.w.value(delivery.BuilderBuildsBucket, key)
if raw == nil {
return catalogue.Manifest{}, false
}
b, err := delivery.DecodeBuilderBuild(raw)
if err != nil || b.Manifest == "" {
return catalogue.Manifest{}, false
}
got := p.w.manifest(ctx, get, b.Manifest)
if got.err != "" {
return catalogue.Manifest{}, false
}
@@ -1,196 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// marksBucket is the assigner's state of what its views family said each state up to.
func marksBucket() string { return assignment.Module() + "_" + views.MarksState }
// viewedTool is one tool over one state of the assigner's.
type viewedTool struct{ tool, state, words string }
// firstViewTools are the assigner's first-view tools (ADR 0621 §1): each answers only its callers list
// (views.CallersEnv), which the manifest writes.
var firstViewTools = []viewedTool{
{"assignments", mma.StateAssignments, "where each module is placed: the seat mesh-module-assigner's assignments, by module."},
{"seat-holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"wants", "wants", "what each node should run: want.<node>.<module>, its fingerprint and the rule that gave it."},
// The seat mesh-wants' asks (design 47 G §717): the asks the assigner put to a person about what a node runs.
{"asks", "asks", "the asks the assigner put to a person about what a node runs, and their answers."},
}
// readTools are the assigner's ordinary read tools, one per state its readers read (design 25 § 12.2).
var readTools = []viewedTool{
{"assigner_read_assignments", mma.StateAssignments, "where each module is placed, by module."},
{"assigner_read_seat_holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
{"assigner_read_findings", assign.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
{"assigner_read_copies", copiesState, "the copies the assigner made of a retired state: when the copy into assignments ended and how many keys it copied."},
}
// viewedStates are the states the views family says, each by its bucket: the six its readers read.
func viewedStates() map[string]string {
return map[string]string{
mma.StateAssignments: plc.AssignmentsBucket(),
mma.StateSeatHolders: plc.SeatHoldersBucket(),
"wants": buckets.Wants(),
assign.FindingsState: assign.FindingsBucket(),
"asks": buckets.Asks(),
"differences": buckets.Differences(),
copiesState: copiesBucket(),
}
}
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
// runtime of its own is connected (ADR 0621 §3).
func viewTools(getenv func(string) string, state func(string) views.Source, now func() time.Time) []stdio.Tool {
marks := state(views.MarksState)
var out []stdio.Tool
for _, t := range firstViewTools {
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now))
}
for _, t := range readTools {
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
}
// Its own first views, of the indexer's manifests, modules and graph (DLV-F3r, ADR 0631 §2): read, and asked again by hand.
return append(out, views.StatusTools(assignment.Module(), "assigner_first_views", "assigner_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
}
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, writer views.WireWriter, errs chan<- error, logf func(string, ...any)) {
process.AwaitBucket(ctx, js, nc, marksBucket(), "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
em := viewsEmitter(js, account, mk, writer, logf)
for _, state := range []string{"asks", "differences", assign.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
state := state
start := func(kv jetstream.KeyValue) {
if err := em.Start(ctx, state, kv, errs); err != nil {
logf("%v: its readers ask its first view at each change until the assigner starts again", err)
}
}
if kv := kvs[viewedStates()[state]]; kv != nil {
start(kv)
continue
}
process.AwaitBucket(ctx, js, nc, viewedStates()[state], "its changes are not said on the views family", logf).Ready(start)
}
})
}
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
// view-unsaid, in its findings, written through its finding writer (review round 2); with none it is said in the log only.
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: assignment.Module(), Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logf}
}
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
func firstViewsBucket() string { return assignment.Module() + "_" + views.StatusState }
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195, A-26): the
// module's views reader adopts each branch's follower, so it asks the seat mesh-catalog's verb `revision` (the trunk's, then
// each branch a registration names), follows its event `catalog-revision`, asks once more on a gap, and past the bound raises
// the finding naming the seat and the verb, cleared by the answer, in the assigner's findings. Its status is read by
// assigner_first_views and asked again by assigner_first_view_again.
type catalogViews struct {
js jetstream.JetStream
account process.Account
call func(string, any) (json.RawMessage, error)
writer views.WireWriter
status *process.AwaitedBucket
logf func(string, ...any)
hub *process.EventHub
h *halves
// candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made;
// before that a candidate's revision is said not taken.
candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error]
// answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers,
// then its views family; nil when it is not followed.
answers *views.Picture
}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
// seconds). A test shortens them.
var firstViewBound, firstViewWait time.Duration
// start adopts the trunk's catalog half through the reader and starts the reader; it answers how many settles to wait for.
func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
reader := &views.ProcessReader{Module: assignment.Module(), Call: iv.call, Logf: iv.logf, Hub: iv.hub, Bound: firstViewBound, Wait: firstViewWait}
if iv.writer != nil {
reader.Findings = func() views.WireWriter { return iv.writer }
}
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
if err := iv.h.follow(); err != nil {
iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err)
}
// Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185).
if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
if f := iv.candidates.Load(); f != nil {
return (*f)(ctx, ev)
}
return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision)
}); err != nil {
iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err)
}
// The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket.
if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil {
iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err)
} else {
iv.answers = pic
}
reader.Start(ctx, iv.js, iv.account, iv.status)
return 1
}
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
// findings state it is said in the log alone.
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
w := heard.Writer{Observer: assignment.Module(), Logf: logf}
if findings.Put != nil {
w.Put = func(_ string, value []byte) error {
var f mcseat.Finding
if err := json.Unmarshal(value, &f); err != nil {
return err
}
return findings.Put(f)
}
}
if findings.Delete != nil {
w.Delete = func(key string) error {
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
return nil // none stands
}
return findings.Delete(key)
}
}
return w
}
@@ -4,35 +4,29 @@ import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strconv"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-module-assigner's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
const (
policiesState = buckets.PoliciesState
policiesState = delivery.AssignerPoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
@@ -49,14 +43,11 @@ type wantsTools struct {
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
// a large state (issue 1953 L5f). Nil reads key by key.
watched *stdio.WatchedStates
// revisions is the kept data assignment-revisions the verbs revision, entry and difference read (design 47 lines
// 1365-1366); nil serves them not.
revisions assigngraph.Objects
}
// wantsEventSubject is the subject of one of the seat's events.
func wantsEventSubject(event string) string {
return "mesh.seat." + meshwants.Seat + ".event." + event
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
}
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
@@ -90,20 +81,25 @@ func (s *wantsTools) direct(state string) (map[string][]byte, error) {
return out, nil
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint: both asked of the indexer's first
// views of modules and manifests (indexerView), never read from its bus state.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]meshcatalog.ModulesByModuleValue, error) {
mods, err := indexerView(s.ask, indexerModules)
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
// under its digest.
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
if s.watched != nil {
return s.watched.Once(manifestsState, digest)
}
raw, _, found, err := s.st.get(manifestsState, digest)
return raw, found, err
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
mods, err := s.values(modulesState)
if err != nil {
return nil, nil, err
}
kept, err := indexerView(s.ask, indexerManifests)
if err != nil {
return nil, nil, err
}
out, held := map[string]catalogue.Manifest{}, map[string]meshcatalog.ModulesByModuleValue{}
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
for name, raw := range mods {
m, err := meshcatalogclient.DecodeModule(raw)
m, err := delivery.DecodeModule(raw)
if err != nil {
continue
}
@@ -111,16 +107,16 @@ func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]mesh
if m.Manifest == "" {
continue
}
mraw, found := kept[m.Manifest]
if !found {
mraw, found, err := s.manifest(m.Manifest)
if err != nil || !found {
continue
}
stored, err := meshcatalogclient.DecodeManifest(mraw)
kept, err := delivery.DecodeManifest(mraw)
if err != nil {
continue
}
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
if man, err := catalogue.Decode([]byte(stored.Manifest)); err == nil {
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
out[name] = man
}
}
@@ -155,21 +151,21 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(buckets.ComposedRead); err != nil {
if values, err := s.values(delivery.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(assign.FindingsState); err != nil {
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
keys := []string{}
@@ -230,8 +226,8 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
}
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, rerr := rec.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
rec := handacts.Recorder{Ask: s.ask}
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
@@ -270,37 +266,37 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
if bus == "" {
return nil, errors.New("the indexer holds no module that provides the bus")
}
newest := held[bus].Fp2
newest := held[bus].FP2
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
h, err := delivery.DecodeHave(raw)
if err != nil || h.Phase == delivery.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
if steps, err := s.ask("seat:"+meshhandacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
out["steps-unread"] = err.Error()
} else {
out["steps"] = steps
}
kept, err := s.values(putsState)
puts, err := s.values(putsState)
if err != nil {
out["puts-unread"] = err.Error()
} else {
mine := map[string]json.RawMessage{}
for k, v := range kept {
if _, module, err := puts.ParseKey(k); err == nil && module == bus {
for k, v := range puts {
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
mine[k] = v
}
}
@@ -309,24 +305,16 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
return out, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>; the tool runner describes them with the
// seat's definition from the seat list (ADR 0635 §1), so the tools carry none.
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
func (s *wantsTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
run := map[string]func(map[string]any) (any, error){
meshwants.VerbUpgrade: func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
meshwants.VerbBus: s.doBus,
}
if s.revisions != nil {
for v, f := range graphVerbs(s.revisions) {
run[v] = f
}
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
"bus": s.doBus,
}
var out []stdio.Tool
for _, v := range meshwants.Verbs() {
if run[v] == nil {
continue // a verb of the seat this module does not serve yet: revision, entry, difference (AS11)
}
out = append(out, stdio.Tool{Name: meshwants.Seat + "." + v, Run: run[v]})
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
}
return out
}
@@ -1,32 +1,25 @@
package main
import (
"encoding/json"
"context"
"errors"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sort"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/keys"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
const noModuleWords = delivery.CandidateNoneNoModule
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
// digest. The watches write it; a pass reads it.
type world struct {
@@ -39,33 +32,14 @@ type world struct {
changes map[string]uint64
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
// ends are the done builds' ends the builder says (build.ended), by module and fingerprint (buildlog.go).
ends builtEnds
// released are the catalog's version-released heard, by module and source revision (buildended.go).
released releasedVersions
// delivered are the assigner's own version-delivered heard and said (delivered.go).
delivered deliveredVersions
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
edges map[string]map[string][]meshcatalog.Edge
// taken is the newest source revision of each watched branch the indexer's take-in.taken named, as heard (orderwaits.go, §298, A-260).
taken takenSources
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
assignmentKeys *mmac.Keys
// first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the
// branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532),
// and never fixed in code (§7).
first string
// presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389).
presence map[string]presenceSaid
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
running map[string]nodeengine.Running
runningHeard bool
// modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14,
// §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog.
modules map[string]map[string]meshcatalog.ModulesByModuleValue
graphs map[string]map[string]graph.Node
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
trunkBuilt map[string]delivery.BuildLogEntry
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
placement *delivery.PlacementKeys
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
sourceGroups map[uint64]string
}
// entryMeta is a value's revision in its bucket and when it was written.
@@ -83,9 +57,8 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(),
running: map[string]nodeengine.Running{},
modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}}
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(),
sourceGroups: map[uint64]string{}}
}
// putEntry is put, keeping the value's revision and the time it was written.
@@ -98,9 +71,47 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
return
}
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
w.sourceGroups[revision] = s.Group
}
}
}
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
// unknown: none, and the move is asked of a person.
func (w *world) release(module, fp string) (string, bool) {
t, ok := w.trunk()[module]
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
return "", false
}
w.mu.Lock()
group := w.sourceGroups[t.FPSrc]
w.mu.Unlock()
key, err := delivery.MergesKey(group)
if group == "" || err != nil {
return "", false
}
raw := w.value(delivery.MergesBucket, key)
if raw == nil {
return "", false
}
merge, err := delivery.DecodeMerge(raw)
if err != nil {
return "", false
}
for _, pull := range merge.Pulls {
if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) {
return group, true
}
}
return "", false
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
@@ -120,28 +131,28 @@ func (w *world) put(bucket, key string, value []byte) {
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
if value == nil {
delete(w.values[bucket], key)
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, nil, 0, true)
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, nil, 0, true)
}
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == plc.AssignmentsBucket() {
if bucket == delivery.PlacementBucket {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.assignmentKeys.Put(key, value, 0, false)
_, _ = w.placement.Put(key, value, 0, false)
}
if bucket == buckets.Have {
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
w.healthy[key] = append(list, h.Fp)
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
w.healthy[key] = append(list, h.FP)
}
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := nodeengineclient.HaveKey(node, module)
key, err := delivery.HaveKey(node, module)
if err != nil {
return ""
}
@@ -167,20 +178,62 @@ func (w *world) read(bucket string) map[string][]byte {
return out
}
// getter reads one key's value; errNotThere when the key does not exist.
type getter func(ctx context.Context, key string) ([]byte, error)
var errNotThere = errors.New("no such key")
// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable.
func (w *world) manifest(digest string) parsed {
w.mu.Lock()
defer w.mu.Unlock()
if p, ok := w.manifests[digest]; ok {
return p
// kvGetter reads keys of a bucket.
func kvGetter(kv jetstream.KeyValue) getter {
return func(ctx context.Context, key string) ([]byte, error) {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, errNotThere
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"}
}
// assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the
// assignments is the one source of where modules run. The controller's assignments are read no more (issue 774).
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
w.mu.Lock()
p, ok := w.manifests[digest]
w.mu.Unlock()
if ok {
return p
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return parsed{err: err.Error()}
}
value, err := get(ctx, key)
if errors.Is(err, errNotThere) {
return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"}
}
if err != nil {
return parsed{err: err.Error()}
}
m, err := delivery.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
// module's value where it is read (world.mesh).
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
} else {
p = parsed{module: m.Module, manifest: man}
}
w.mu.Lock()
w.manifests[digest] = p
w.mu.Unlock()
return p
}
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
func (w *world) assigned() map[string][]string {
placed, _ := w.placed()
out := make(map[string][]string, len(placed))
@@ -190,23 +243,23 @@ func (w *world) assigned() map[string][]string {
return out
}
// placed is each node's modules in the seat's assignments (ADR 0567 §1), and whether any assignments key was read: a key
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
for node, p := range w.assignments() {
out[node] = mmac.Assigned(p)
for node, p := range w.placements() {
out[node] = p.Assigned()
}
return out, len(w.read(plc.AssignmentsBucket())) > 0
return out, len(w.read(delivery.PlacementBucket)) > 0
}
// assignments is every node's assignments, assembled from its keys in either form (mmac.Keys).
func (w *world) assignments() map[string]mma.Assignments {
out, _ := w.assignmentKeys.AllRevisions()
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
func (w *world) placements() map[string]delivery.Placement {
out, _ := w.placement.AllRevisions()
return out
}
// placedNodes is every node the seat's assignments names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
@@ -217,187 +270,66 @@ func (w *world) placedNodes() map[string]bool {
return out
}
// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184).
func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue {
w.mu.Lock()
defer w.mu.Unlock()
return maps.Clone(w.modules[branch])
}
// graphOf is the module graph of the catalog half of branch.
func (w *world) graphOf(branch string) graph.Graph {
w.mu.Lock()
defer w.mu.Unlock()
return graph.Graph(maps.Clone(w.graphs[branch]))
}
// branches are the watched branches the catalog half is held of, the trunk's first.
func (w *world) branches() []string {
w.mu.Lock()
defer w.mu.Unlock()
out := slices.Sorted(maps.Keys(w.modules))
if i := slices.Index(out, w.first); i > 0 {
out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first)
}
return out
}
// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's
// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it.
func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue {
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, b := range w.branches() {
for name, v := range w.trunkOf(b) {
if _, ok := out[name]; !ok {
out[name] = v
}
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
for key, value := range w.read(delivery.ModulesBucket) {
name, err := delivery.ParseModulesKey(key)
if err != nil {
continue
}
if m, err := delivery.DecodeModule(value); err == nil {
out[name] = m
}
}
return out
}
// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the
// value names.
func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) {
w.mu.Lock()
defer w.mu.Unlock()
if w.modules[branch] == nil {
w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{}
}
w.modules[branch][module], w.graphs[branch][module] = v, n
if m != nil && v.Manifest != "" {
w.manifests[v.Manifest] = *m
}
}
// dropModule takes a module out of branch's catalog half.
func (w *world) dropModule(branch, module string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.modules[branch], module)
delete(w.graphs[branch], module)
delete(w.edges[branch], module)
}
// putEdges keeps the edges into module's artifacts that branch's catalog entry names.
func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) {
w.mu.Lock()
defer w.mu.Unlock()
if w.edges == nil {
w.edges = map[string]map[string][]meshcatalog.Edge{}
}
if w.edges[branch] == nil {
w.edges[branch] = map[string][]meshcatalog.Edge{}
}
w.edges[branch][module] = edges
}
// edgesOf is the edges into module's artifacts in branch's catalog half.
func (w *world) edgesOf(branch, module string) []meshcatalog.Edge {
w.mu.Lock()
defer w.mu.Unlock()
return w.edges[branch][module]
}
// holdsBranch says whether the catalog half of branch has been taken in (even empty).
func (w *world) holdsBranch(branch string) bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.modules[branch] != nil
}
// records is each node's record in its own have key (`<node>.node`): the nodes an assignments proposal is answered by,
// with those assignments names (ADR 0593 §8).
func (w *world) records() map[string]nodeengine.NodeRecord {
out := map[string]nodeengine.NodeRecord{}
for key, value := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
// with those placement names (ADR 0593 §8).
func (w *world) records() map[string]delivery.NodeRecord {
out := map[string]delivery.NodeRecord{}
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
return out
}
// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389).
type presenceSaid struct {
away bool
at time.Time
}
// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two
// streams, read whole in either order, leave each machine as its last event says.
func (w *world) heardPresence(machine string, away bool, at time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
if w.presence == nil {
w.presence = map[string]presenceSaid{}
}
if was, ok := w.presence[machine]; ok && at.Before(was.at) {
return
}
w.presence[machine] = presenceSaid{away: away, at: at}
}
// awayNow is every machine whose last presence event heard is machine-away.
func (w *world) awayNow() map[string]bool {
w.mu.Lock()
defer w.mu.Unlock()
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
for m, p := range w.presence {
if p.away {
out[m] = true
}
}
return out
}
// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to
// the seat's verb is heard (novox/hq ADR 0668 §3).
func (w *world) seatHeard(seat string) bool {
online := w.online()
for _, value := range w.read(plc.SeatHoldersBucket()) {
var sh mma.SeatHolders
if json.Unmarshal(value, &sh) != nil || sh.Seat != seat {
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
for _, h := range sh.Holders {
if online[h.Node] {
return true
}
if p, err := delivery.DecodePresence(value); err == nil {
out[node] = p.Online
}
}
return false
}
// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its
// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2).
func (w *world) online() map[string]bool {
away := w.awayNow()
out := map[string]bool{}
for _, node := range w.knownNodes() {
out[node] = !away[node]
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]mwc.Want {
out := map[string]map[string]mwc.Want{}
for key, value := range w.read(buckets.Wants()) {
node, module, err := mwc.ParseWantKey(key)
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
for key, value := range w.read(delivery.AssignerWantsBucket) {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
continue
}
want, err := mwc.DecodeWant(value)
want, err := delivery.DecodeWant(value)
if err != nil {
continue
}
if out[module] == nil {
out[module] = map[string]mwc.Want{}
out[module] = map[string]delivery.Want{}
}
out[module][node] = want
}
@@ -406,24 +338,32 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
n.Online = w.online()
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
n.Online[node] = p.Online
}
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil {
continue
}
if isNode {
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
n.Reads[node] = h.Record.ManifestReads
}
continue
}
if h, err := nodeengineclient.DecodeHave(value); err == nil {
if h, err := delivery.DecodeHave(value); err == nil {
if n.Haves[node] == nil {
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
n.Haves[node] = map[string]delivery.Have{}
}
n.Haves[node][module] = h
}
@@ -454,7 +394,7 @@ func (w *world) value(bucket, key string) []byte {
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
// manifest but the controller's reads the controller's grants.
func (w *world) grantsBucket() string { return buckets.Grants }
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
@@ -464,73 +404,41 @@ func (w *world) has(bucket, key string) bool {
return ok
}
// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181).
// built says whether the builder's build of a module at a fingerprint is done.
func (w *world) built(module, fp string) bool {
e, ok := w.ends.endOf(module, fp)
return ok && e.Outcome == builds.BuildDone
}
// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G
// §575, A-162): the words of a hold under rule 1.
func (w *world) buildStanding(module, fp string) (failed bool, words string) {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed {
return true, e.Error
}
return false, ""
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR
// 0618 §1, A-181): nil when it is not done or names none.
func (w *world) buildNeeds(module, fp string) []string {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone {
return e.Needs
}
return nil
}
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
func (w *world) recordReads(node string) ([]string, bool) {
r, ok := w.records()[node]
return r.ManifestReads, ok
}
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
func (w *world) knownNodes() []string {
set := map[string]bool{}
for node := range w.placedNodes() {
set[node] = true
}
for node := range w.records() {
set[node] = true
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return false
}
w.mu.Lock()
for node := range w.presence {
set[node] = true
}
value, ok := w.values[delivery.BuilderBuildsBucket][key]
w.mu.Unlock()
return slices.Sorted(maps.Keys(set))
if !ok {
return false
}
b, err := delivery.DecodeBuilderBuild(value)
return err == nil && b.State == delivery.BuildDone
}
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
if !w.placedNodes()[node] {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (mdc.Answer, bool) {
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[buckets.Answers][key]
value, ok := w.values[delivery.AnswersBucket][key]
w.mu.Unlock()
if !ok {
return mdc.Answer{}, false
return delivery.Answer{}, false
}
a, err := mdc.DecodeAnswer(value)
a, err := delivery.DecodeAnswer(value)
return a, err == nil
}
@@ -551,30 +459,30 @@ func (w *world) changed(bucket, key string) uint64 {
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
key, err := nodeengineclient.HaveNodeKey(node)
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
if err != nil {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
raw, ok := w.read(buckets.Have)[key]
raw, ok := w.read(delivery.HaveBucket)[key]
if !ok {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
h, err := nodeengineclient.DecodeHaveNode(raw)
h, err := delivery.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
Words: "the node composes its own"}, true
}
// unplacedNodes are the nodes whose engine has a have and that no assignments names: they get no
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
// want, since nothing composes them.
func (w *world) unplacedNodes() []string {
placed := w.placedNodes()
var out []string
for key := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
for key := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
}
@@ -582,31 +490,3 @@ func (w *world) unplacedNodes() []string {
sort.Strings(out)
return out
}
// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted.
func (w *world) trunkName() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.first
}
// setTrunk says which watched branch is the first.
func (w *world) setTrunk(branch string) {
w.mu.Lock()
defer w.mu.Unlock()
w.first = branch
}
// placedBranches is, by module and node, the watched branch each placement names ("" when it names none).
func (w *world) placedBranches() map[string]map[string]string {
out := map[string]map[string]string{}
for node, a := range w.assignments() {
for _, m := range a.Modules {
if out[m.Module] == nil {
out[m.Module] = map[string]string{}
}
out[m.Module][node] = m.Branch
}
}
return out
}
@@ -2,17 +2,12 @@ package main
import (
"context"
"mesh-delivery-assigner/internal/buckets"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// fakeManifests is the indexer's manifests bucket in memory.
@@ -20,8 +15,8 @@ type fakeManifests map[string][]byte
func (f fakeManifests) add(t *testing.T, module, body string) string {
t.Helper()
digest := meshcatalogclient.ManifestDigest([]byte(body))
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
digest := delivery.ManifestDigest([]byte(body))
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
if err != nil {
t.Fatal(err)
}
@@ -36,24 +31,33 @@ func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
return nil, errNotThere
}
func fp(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
for m := range modules {
p.Modules = append(p.Modules, m)
}
slices.Sort(p.Modules)
return p
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(buckets.Have, "gone"); got != 0 {
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{Accounts: []string{"ace.web"}})
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name string
inPart bool
@@ -62,11 +66,11 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
{"the part's grant", true, true},
{"no grant", false, false},
} {
p := newPasser(kvSet{}, time.Now())
p := newPasser(buckets{}, time.Now())
if row.inPart {
p.w.put(buckets.Grants, gkey, grant)
p.w.put(delivery.PartGrantsBucket, gkey, grant)
}
if got := p.w.grantsBucket(); got != buckets.Grants {
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
@@ -74,41 +78,3 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
}
}
}
// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it.
func putModuleRaw(w *world, branch, module string, raw []byte) {
v, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
panic(err)
}
w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil)
}
// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the
// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main".
func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) {
w := newWorld()
w.setTrunk("develop")
for _, b := range []string{"beta", "develop", "alpha"} {
w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil)
}
if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" {
t.Errorf("branches = %v, want the first watched first, then by name", got)
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(src), `"main"`) {
t.Errorf("%s names the branch main in code (design 47 G §507)", f)
}
}
}
+7 -93
View File
@@ -3,110 +3,24 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.1 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.2 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.16.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk => ../mesh-delivery-sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient => ../mesh-delivery-contracts/go/clients/meshcatalogclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient => ../mesh-delivery-contracts/go/clients/meshdeliveryclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog => ../mesh-delivery-contracts/go/seats/meshcatalog
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery => ../mesh-delivery-contracts/go/seats/meshdelivery
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks => ./asks
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents => ../mesh-delivery-teller/tellerevents
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate => ../mesh-delivery-view/viewstate
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient => ../mesh-delivery-contracts/go/clients/meshdeliverycheckerclient
require git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient => ../mesh-delivery-contracts/go/clients/meshpipelinerunnerclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner => ../mesh-delivery-contracts/go/seats/meshpipelinerunner
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
+10 -8
View File
@@ -1,13 +1,15 @@
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
@@ -21,5 +23,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
@@ -1,140 +0,0 @@
// Package acting is the assigner's wait for an acting part's new version (design 47 G §266, §268, §276): an entry of the assignment
// graph whose edge of kind acting names a step the assigner holds (composing, placing) waits until the acting part's version runs
// on the node of the edge, with the bound of the rule that gave the edge, or the step's default. The package holds the rules and
// the kept record; the assigner's process reads what runs, says the events and keeps the records in its state `waits`.
package acting
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// HowNow is the `how` of assigner_release for a wait for an acting part: the entry is wanted now, with the version running.
const HowNow = "now"
// The ways a wait ends, as `wait-ended` says them.
const (
HowRan = "ran"
HowReleased = "released"
HowReplaced = "replaced"
)
// Prefix opens the key of an acting wait in the state, apart from the outside waits' `<node>.<module>.<fingerprint hex>`.
const Prefix = "acting."
// OrderPrefix opens the key of an order wait (design 47 G §298, A-259): a want held for the order of a delivery that crosses a form change,
// kept beside the acting waits in the state `waits`, as `order.<node>.<module>.<part>`.
const OrderPrefix = "order."
// The three order edges of §298, as a kept wait's Order names them.
const (
// OrderBackup: the new versions of the node-backup holder and of the holder of the bus module's seat run before a module whose kept
// data's form changes is moved.
OrderBackup = "backup-and-broker-first"
// OrderEngineLast: at a revert a node's node-engine goes back last on that node.
OrderEngineLast = "node-engine-last"
// OrderTakenIn: the reverted assigner and gate are wanted only once the reverted indexer took in the trunk's head.
OrderTakenIn = "taken-in-first"
)
// Held says whether the assigner holds the entries of step: composing and placing (§276's table).
func Held(step string) bool { return step == seatdef.StepCompose || step == seatdef.StepPlace }
// Wait is one wait as kept: Module's entry on Node waits for Part's version Fp to run on PartNode, in Step.
type Wait struct {
V int `json:"v"`
Step string `json:"step"`
Node string `json:"node"`
Module string `json:"module"`
Part string `json:"part"`
PartNode string `json:"part-node"`
Fp string `json:"fp"`
Entry string `json:"entry"`
Since time.Time `json:"since"`
ClockFrom time.Time `json:"clock-from"`
Bound string `json:"bound"`
// Released is a release by hand: the entry moves on with the part's old version; EndSaid says wait-ended was said for it.
// Order is which of §298's edges the wait stands on; empty for an acting wait.
Order string `json:"order,omitempty"`
Released bool `json:"released,omitempty"`
EndSaid bool `json:"end-said,omitempty"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
}
// Key is the key of a wait in the state.
func Key(node, module, part string) string { return Prefix + node + "." + module + "." + part }
// OrderKey is the key of an order wait in the state.
func OrderKey(node, module, part string) string {
return OrderPrefix + node + "." + module + "." + part
}
// IsOrder says a kept value is an order wait's record.
func IsOrder(key string) bool { return strings.HasPrefix(key, OrderPrefix) }
// Is says a kept value is an acting wait's record.
func Is(key string) bool { return strings.HasPrefix(key, Prefix) }
// Encode is w as JSON.
func Encode(w Wait) ([]byte, error) { w.V = 1; return json.Marshal(w) }
// Decode reads a kept wait.
func Decode(raw []byte) (Wait, error) {
var w Wait
if err := json.Unmarshal(raw, &w); err != nil {
return w, err
}
if w.V != 1 || w.Step == "" || w.Node == "" || w.Module == "" || w.Part == "" {
return w, fmt.Errorf("acting: %q is not a kept acting wait", raw)
}
return w, nil
}
// BoundOf is the bound of the edge's wait: its rule's, else its step's default (seatdef.ActingRule.Wait).
func BoundOf(e meshwants.Edge) time.Duration {
return seatdef.ActingRule{Step: e.Step, Bound: e.Bound}.Wait()
}
// Pending are the acting edges of the assigner's steps whose part's version does not run yet: runs says the edge's part runs
// its version on the edge's node.
func Pending(edges []meshwants.Edge, runs func(e meshwants.Edge) bool) []meshwants.Edge {
var out []meshwants.Edge
for _, e := range edges {
if e.Kind == "acting" && Held(e.Step) && !runs(e) {
out = append(out, e)
}
}
return out
}
// Body is the wait as the seat mesh-wants says it.
func Body(w Wait, owner string) meshwants.Wait {
since, from := w.Since.UTC(), w.ClockFrom.UTC()
return meshwants.Wait{Step: w.Step, Owner: owner, Node: w.Node, Module: w.Module, Part: w.Part, Fp: w.Fp, Since: since, ClockFrom: &from, Bound: w.Bound,
Words: words(w)}
}
func words(w Wait) string {
switch w.Order {
case OrderBackup:
return fmt.Sprintf("%s on %s moves its kept data across a form change: it waits in %s for %s at %s to run on %s, which the restore point needs", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
case OrderEngineLast:
return fmt.Sprintf("%s on %s goes back last on its node: it waits in %s for %s at %s to run on %s, restored", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
case OrderTakenIn:
return fmt.Sprintf("%s on %s goes back only once the indexer took in the trunk's head: it waits in %s for source revision %s to be taken in", w.Module, w.Node, w.Step, w.Fp)
}
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
}
func short(fp string) string {
if len(fp) > 19 {
return fp[:19]
}
return fp
}
@@ -1,69 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
func assignerAsk(asking int, a oc.Ask) asks.Ask {
raw, err := json.Marshal(a)
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
}
return asks.Ask{Asking: asking, Ask: raw}
}
// AskOf is the router's ask an asks-state value carries.
func AskOf(a asks.Ask) (oc.Ask, error) {
var ask oc.Ask
if err := json.Unmarshal(a.Ask, &ask); err != nil {
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
}
return ask, nil
}
// EncodeAsk is asks.EncodeAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
// the ask's id is meshdeliveryclient.AskingID(key, asking), and it offers at least one option.
func EncodeAsk(key string, a asks.Ask) ([]byte, error) {
if _, err := checkedAsk(key, a); err != nil {
return nil, err
}
return asks.EncodeAsk(key, a)
}
func checkedAsk(key string, a asks.Ask) (oc.Ask, error) {
ask, err := occ.Carried(a.Ask, meshdeliveryclient.AskingID(key, a.Asking))
if err != nil {
return oc.Ask{}, fmt.Errorf("assign: %w", err)
}
return ask, nil
}
// DecodeAsk reads the ask at key (asks.DecodeAsk) with the router's ask it carries.
func DecodeAsk(key string, data []byte) (asks.Ask, oc.Ask, error) {
a, err := asks.DecodeAsk(key, data)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
ask, err := checkedAsk(key, a)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
return a, ask, nil
}
// ExpiresAt is when the ask ends unanswered; the zero time for an ask that names none.
func ExpiresAt(a oc.Ask) time.Time {
t, err := occ.Time(a.Expires)
if err != nil {
return time.Time{}
}
return t
}
@@ -2,40 +2,34 @@ package assign
import (
"fmt"
"mesh-delivery-assigner/internal/assignment"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Options: []oc.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the version stays held",
About: key,
})
}}
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
@@ -45,86 +39,44 @@ const (
)
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's
// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the
// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []oc.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module stays as it is on the node",
About: key,
})
}
// The options of the outside ask (design 47 G §706).
const (
OptionGoOn = "go-on"
OptionFailed = "failed"
)
// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
Options: []oc.Option{
{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the delivery keeps waiting",
About: about,
})
}
// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
Severity: "warning",
Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
}}
}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = occ.NeedsYou + c.Needs + ". " + what
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
@@ -133,22 +85,22 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Con
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
key := assignment.Module() + ".failed-alone." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
@@ -156,28 +108,28 @@ func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
key := assignment.Module() + ".neither." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
key := delivery.AssignerModule + ".neither." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) oc.Condition {
func NeitherLeftCondition(node, module, why string) asks.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
@@ -186,7 +138,7 @@ func NeitherLeftCondition(node, module, why string) oc.Condition {
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) oc.Condition {
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
@@ -196,91 +148,37 @@ func LiftedCondition(module, fp string) oc.Condition {
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = mwc.BackupAgain
OptionBackupSkip = mwc.BackupSkip
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
)
// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
const (
OptionRevertHold = "hold"
OptionRevertLive = mwc.RevertLive
)
// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
const KindRevertHeld = "revert-held"
// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
// the asking, 0 the first, as the backup ask's.
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
module, node, short(fp), kept, form, words, module),
Options: []oc.Option{
{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the module keeps running its newer build",
About: key,
})
}
// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
func RevertAnswerWords(option string, asking int) string {
return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
}
// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
// operator's one message is the ask.
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
module, reason, kept, form, module),
Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
}
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
// asked again, for the rewritten key carried the id the router already kept).
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + need.Words,
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
Options: []oc.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the node keeps what it runs",
About: key,
})
}}
}
// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
@@ -293,12 +191,8 @@ var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a asks.Ask) string {
ask, err := AskOf(a)
if err != nil {
return ""
}
if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
func BackupAskDigest(a delivery.AssignerAsk) string {
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
return m[1]
}
return ""
@@ -313,34 +207,34 @@ func BackupAnswerWords(option string, asking int) string {
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Options: []oc.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module keeps running what it has",
About: key,
})
}}
}
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
needs := "answer whether to back up again or go on without a backup"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}
@@ -1,250 +1,75 @@
package assign
import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/puts"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
// its key, of a kind the view may answer.
func TestTheHoldAskIsValid(t *testing.T) {
t.Run("hold", theHoldAskIsValid)
t.Run("restore", theRestoreAskIsValid)
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
}
// Issue 1966, note 151: an ask the teller says ended at the operator channel without an answer (its event ask-ended) is
// written again under a new asking, an id of its own and a new expiry, one row per kind: the rules' asks are; the
// authorising ask of a proposal (ADR 0567 §5, §11) and a put's record ask (ADR 0546) are not, nor an asking the key no longer
// holds, an end asking again would not change, or a body that does not read.
func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
now := asked.Add(8 * 24 * time.Hour)
digest := "sha256:" + strings.Repeat("e", 64)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
assignmentsKey, _ := asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")
need := RecordNeed{Node: "novox", Digest: digest, Judgement: JudgedMovesData}
record := RecordAsk(recordKey, "novox", "nats", fNew, need, 0, asked) // written before #195: no asking
put := puts.Put{FP: fNew, Why: "w", At: asked, Step: puts.Person, Ask: recordKey}
byPut := RecordAsk(recordKey, "novox", "nats", fNew, need, PutAsking(put), asked)
other := puts.Put{FP: fNew, Why: "w", At: asked.Add(time.Hour), Step: puts.Person, Ask: recordKey}
assignments := assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(assignmentsKey, AskingSeconds(asked)), Headline: "h",
Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(occ.ApproveLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: OptionYes, Label: "Yes", Does: "d", Level: occ.Approve}}})
for _, c := range []struct {
name string
key string
a *asks.Ask // the ask the key holds; nil for none
ended string // the id of the asking that ended; "" for the one the key holds
outcome string
put *puts.Put
why string // "" when it is written again
}{
{name: "hold", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeExpired},
{name: "restore", key: restoreKey, a: ptr(RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeCancelled},
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, digest, "w", 0, asked)), outcome: occ.OutcomeCancelled},
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
{name: "record", key: recordKey, a: &record, outcome: occ.OutcomeCancelled},
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
{name: "another asking stands", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(now), now)),
ended: meshdeliveryclient.AskingID(holdKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
{name: "taken away", key: holdKey, ended: "hold-x", outcome: occ.OutcomeExpired, why: "taken away"},
{name: "refused", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
{name: "replaced", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
} {
t.Run(c.name, func(t *testing.T) {
var current []byte
ended := c.ended
if c.a != nil {
var err error
if current, err = EncodeAsk(c.key, *c.a); err != nil {
t.Fatal(err)
}
if ended == "" {
ended = mustAsk(t, *c.a).ID
}
}
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: c.key, Ask: ended, Outcome: c.outcome})
if err != nil {
t.Fatal(err)
}
e, err := tellerevents.DecodeEnded(body)
if err != nil {
t.Fatal(err)
}
value, why := AskedAgain(e, current, c.put, now)
if c.why != "" {
if value != nil || !strings.Contains(why, c.why) {
t.Fatalf("AskedAgain = %s, %q; want nothing written, %q", value, why, c.why)
}
return
}
if why != "" {
t.Fatalf("AskedAgain says %q, want the ask written again", why)
}
a, ask, err := DecodeAsk(c.key, value)
if err != nil {
t.Fatal(err)
}
if a.Asking <= c.a.Asking || ask.ID != meshdeliveryclient.AskingID(c.key, a.Asking) || ask.ID == ended {
t.Errorf("asking %d under %s, want a new asking above %d under an id of its own, not %s", a.Asking, ask.ID, c.a.Asking, ended)
}
if err := occ.Check(ask, now); err != nil || !ExpiresAt(ask).Equal(now.Add(AskLasts).Truncate(time.Second)) {
t.Errorf("the router refuses the ask written again (%v), or it lapses at %s, want %s", err, ExpiresAt(ask), now.Add(AskLasts))
}
})
}
for _, raw := range []string{`{"v":1,"key":"k","ask":"a"}`, `{"v":1,"key":"k","ask":"a","outcome":"expired","more":1}`} {
if _, err := tellerevents.DecodeEnded([]byte(raw)); err == nil {
t.Errorf("DecodeEnded(%s) takes a body that is not an ended ask", raw)
}
}
}
// Issue 1966 note 155, ADR 0668 §4: an ended ask the teller said before the assigner's consumer existed is not delivered to
// it, so at bind the assigner reads the operator-channel seat's asks: the query reaches back to the earliest held asking,
// and of the answer only the held askings that lapsed or were cancelled are taken, as the teller's event says them; an
// answered or another asker's ask, and one the key no longer holds, are not.
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)
restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking
held := map[string][]byte{}
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
v, err := EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
}
held[key] = v
}
if _, ok := ChannelAsksQuery(nil); ok {
t.Error("with no ask held the channel is asked")
}
q, ok := ChannelAsksQuery(held)
if !ok || q.State != "ended" || q.Bound != ChannelAsksBound || q.Since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) {
t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339))
}
at := func(key string, a asks.Ask, state string) string {
return fmt.Sprintf(`{"key":"t.%s","state":%q}`, mustAsk(t, a).ID, state)
}
answer := `{"asks":[` + strings.Join([]string{at(holdKey, hold, occ.OutcomeExpired), at(restoreKey, restore, occ.OutcomeChosen),
at(backupKey, backup, occ.OutcomeCancelled), `{"key":"t.hold-gone","state":"expired"}`, `{"key":"open","state":"open"}`}, ",") + `],"cut":"c"}`
ended, cut, err := EndedAtChannel(held, []byte(answer))
want := []tellerevents.Ended{{Key: backupKey, Ask: mustAsk(t, backup).ID, Outcome: occ.OutcomeCancelled},
{Key: holdKey, Ask: mustAsk(t, hold).ID, Outcome: occ.OutcomeExpired}}
slices.SortFunc(want, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
if err != nil || cut != "c" || !slices.Equal(ended, want) {
t.Errorf("EndedAtChannel = %+v, %q, %v; want %+v and the answer's cut", ended, cut, err, want)
}
if _, _, err := EndedAtChannel(held, []byte(`[1]`)); err == nil {
t.Error("an answer that does not read is taken")
}
}
func ptr(a asks.Ask) *asks.Ask { return &a }
func theHoldAskIsValid(t *testing.T) {
key, err := asks.HoldAskKey("mail", fNew)
key, err := delivery.HoldAskKey("mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := EncodeAsk(key, again); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0))
other := good
other.Asking = 3 // its id is the asking 0's
empty := good
ask := mustAsk(t, good)
ask.Options = nil
empty.Ask = assignerAsk(0, ask).Ask
for name, a := range map[string]asks.Ask{"another id": other, "no option": empty, "no ask": {}} {
if _, err := EncodeAsk(key, a); err == nil {
t.Errorf("%s: written", name)
}
}
})
}
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func theRestoreAskIsValid(t *testing.T) {
key, err := asks.RestoreAskKey("ace", "mail", fNew)
func TestTheRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", 2, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range mustAsk(t, a).Options {
for _, o := range a.Ask.Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation)
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
}
}
@@ -258,13 +83,13 @@ func TestTheAssignersConditions(t *testing.T) {
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key {
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
@@ -272,10 +97,10 @@ func heldConditionNamesModuleNodeAndWords(t *testing.T) {
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
@@ -287,37 +112,38 @@ func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
func saidConditionsAreFindings(t *testing.T) {
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
for _, c := range []struct {
raised, cleared oc.Condition
raised, cleared asks.Condition
key string
}{
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"},
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]mcseat.Finding{}, []string{}, 0
f := Findings{Put: func(f mcseat.Finding) error {
puts[views.FindingKey(f.Scope, f.ID, f.Kind)] = f
return nil
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
f := Findings{Put: func(k string, v []byte) error {
got, err := health.DecodeFinding(v)
puts[k] = got
return err
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
say := f.Say(func(oc.Condition) error { said++; return nil })
say := f.Say(func(asks.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
t.Fatalf("%s: %v", c.key, err)
}
got, ok := puts[c.key]
if !ok || got.Severity != c.raised.Severity || got.Headline != c.raised.Headline || got.Observer != assignment.Module() ||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
}
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
}
if err := (Findings{}).Say(func(oc.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
if err := (Findings{}).Say(func(asks.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
}
}
if _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
t.Error("a condition not the assigner's is a finding")
}
}
@@ -326,41 +152,36 @@ func saidConditionsAreFindings(t *testing.T) {
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := asks.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, 2, now)
bkey, err := asks.BackupAskKey("ace", "mail", fNew)
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
v := RevertAsk(vkey, "ace", "mail", fNew, "records", "3", "restore point missing: none of form 3", 0, now)
for key, a := range map[string]asks.Ask{rkey: r, bkey: b, vkey: v} {
if err := occ.Check(mustAsk(t, a), now); err != nil {
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
if err := a.Ask.Check(now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", mustAsk(t, r))
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", r.Ask)
}
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
t.Errorf("the backup ask = %+v", mustAsk(t, b))
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
t.Errorf("the backup ask = %+v", b.Ask)
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
@@ -372,68 +193,45 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
t.Errorf("two answers of the same option are one want value")
}
// design 47 G §710: the revert ask names the build, the kept data, its form and the node's words, and offers hold and live;
// its finding is delivery.<node>.<module>.revert-held, a warning that needs nothing, the ask being the one message.
if va := mustAsk(t, v); va.Headline != "The revert of mail on ace has no restore point" || va.Options[0].ID != OptionRevertHold ||
va.Options[1].ID != mwc.RevertLive || len(va.Options) != 2 || va.OnExpiry != "the module keeps running its newer build" ||
!strings.Contains(va.Explanation, "records") || !strings.Contains(va.Explanation, "form 3") || !strings.Contains(va.Explanation, short(fNew)) ||
!strings.Contains(va.Explanation, "Started on the live data, the older build reads data written in the newer form.") {
t.Errorf("the revert ask = %+v", va)
}
if f := RevertHeldFinding("ace", "mail", "records", "3", "restore point missing").Finding(now); views.FindingKey(f.Scope, f.ID, f.Kind) != "delivery.ace.mail.revert-held" ||
f.Severity != "warning" || f.Needs != "Nothing for you to do." ||
f.Summary != "The revert of mail on ace is held: restore point missing (records, form 3); the ask about it waits for your answer" {
t.Errorf("the revert finding = %+v", f)
}
if RevertAnswerWords(OptionRevertLive, 1) != `the operator answered "live" to the revert ask (asking 2)` {
t.Errorf("the words of a live answer: %q", RevertAnswerWords(OptionRevertLive, 1))
}
c := BackupCondition("ace", "mail", fNew, "no space left", true)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
t.Fatalf("bus %v: %v", bus, err)
}
if err := occ.Check(mustAsk(t, a), now); err != nil {
if err := a.Ask.Check(now); err != nil {
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
}
if _, err := EncodeAsk(rkey, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := viewstate.ViewAnswerFits(viewstate.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
yes := mustAsk(t, a).Options[0]
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
// acknowledge), as it did through the sdk before.
if viewed == nil {
if o, ok := occ.OptionOf(mustAsk(t, a), OptionYes); ok && o.Level != "" && o.Level != occ.Acknowledge {
viewed = fmt.Errorf("only a channel that proves who answered takes the option %s", o.ID)
}
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
yes := a.Ask.Options[0]
switch {
case !strings.Contains(mustAsk(t, a).Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, mustAsk(t, a).Explanation)
case bus && (yes.Level != occ.Approve || yes.Binds == "" || viewed == nil || expiresAt(t, a).After(now.Add(occ.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, expiresAt(t, a))
case !bus && (yes.Level != occ.Acknowledge || viewed != nil):
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
}
later := put
later.At = now.Add(time.Minute)
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
if mustAsk(t, b).ID == mustAsk(t, a).ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, mustAsk(t, b).ID, mustAsk(t, a).ID)
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dgst)); err != nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
t.Errorf("bus: the yes performs another composition's act")
}
}
@@ -442,30 +240,15 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := occ.EncodeCondition(c); err != nil {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key {
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// mustAsk is the router's ask an asks-state value carries.
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
t.Helper()
ask, err := AskOf(a)
if err != nil {
t.Fatal(err)
}
return ask
}
func expiresAt(t *testing.T, a asks.Ask) time.Time {
t.Helper()
return ExpiresAt(mustAsk(t, a))
}
@@ -1,23 +0,0 @@
package assign
import (
"mesh-delivery-assigner/internal/assignment"
"strings"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// SeatWaitsCondition is the assigner's word on the seat mesh-module-assigner while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) oc.Condition {
key := assignment.Module() + ".seat-states"
subject := oc.ConditionSubject{Scope: "module", ID: assignment.Module()}
if len(unbound) == 0 {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, Resolved: "the seat mesh-module-assigner holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: occ.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -1,45 +0,0 @@
package assign
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
// judgement); neither and unchanged are the assigner's own.
type Judgement = string
// The judgements of a move.
const (
// JudgedRecreates is a move that recreates a container.
JudgedRecreates = nec.JudgedRecreates
// JudgedMovesData is a move that gives the module's data another place.
JudgedMovesData = nec.JudgedMovesData
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
JudgedNeither = "neither"
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
JudgedCannot = nec.JudgedCannot
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
// changes nothing on the node, so it needs no person's yes.
JudgedUnchanged = "unchanged"
)
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
// the module). Words say why, and are required for a move that cannot be judged.
//
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
type ComposedDigest struct {
V int `json:"v"`
Digest string `json:"digest"`
Judgement Judgement `json:"judgement"`
Against string `json:"against,omitempty"`
Words string `json:"words,omitempty"`
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
// record hold (ADR 0539 clause 7).
UserListOnly bool `json:"user-list-only,omitempty"`
Policy string `json:"policy,omitempty"`
PolicyFrom string `json:"policy-from,omitempty"`
PolicyWhy string `json:"policy-why,omitempty"`
}
@@ -1,139 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/puts"
)
// AskLasts is how long an ask of the assigner's rules stands at the operator channel before it lapses unanswered: the hold,
// restore, record, backup, outside and revert asks.
const AskLasts = 7 * 24 * time.Hour
// writtenAgainOnEnd is, per kind of the assigner's asks, whether an asking that ended at the channel without an answer while
// its key stands is written again under a new asking (issue 1966, note 151), or why not. A rule's ask waits for a person
// as long as its key stands, so it is asked again; the authorising ask of a proposal ends the proposal when it lapses
// (ADR 0567 §5) and is asked again by the seat's verb `again` (§11).
var writtenAgainOnEnd = map[string]string{
asks.AskKindHold: "",
asks.AskKindRestore: "",
asks.AskKindBackup: "",
asks.AskKindOutside: "", // raised again at once while the wait stands (design 47 G §706); the wait's end takes the key away
asks.AskKindRecord: "",
asks.AskKindRevert: "", // raised again at once while the hold stands (design 47 G §710); one answered hold is not asked again
asks.AskKindAssignments: "a proposal's authorising ask that lapsed ends the proposal (ADR 0567 §5); the verb again asks it again (§11)",
}
// AskedAgain is the ask at key written again after the teller said its asking ended at the channel without an answer (the
// teller's event ask-ended): the value to write by compare-and-set over current, or why nothing is written. Nothing is written
// for an outcome other than a lapse or a cancellation, for a key that holds another asking now (an answer to it, or a newer
// asking, came since), for a kind its own rule ends (writtenAgainOnEnd), and for a record ask a put asked (put, nil for none:
// the put at the record ask's node and module), since its put is refused when it lapses (ADR 0546). The new asking is
// AskingSeconds(now), above the old one, under an id of its own, standing AskLasts from now.
func AskedAgain(e tellerevents.Ended, current []byte, put *puts.Put, now time.Time) ([]byte, string) {
if e.Outcome != occ.OutcomeExpired && e.Outcome != occ.OutcomeCancelled {
return nil, fmt.Sprintf("it ended %s, which asking again would not change", e.Outcome)
}
k, err := asks.ParseAnyAskKey(e.Key)
if err != nil {
return nil, err.Error()
}
why, known := writtenAgainOnEnd[k.Kind]
if !known {
return nil, "the assigner asks no " + k.Kind + " ask"
}
if why != "" {
return nil, why
}
if current == nil {
return nil, "the key is taken away: nothing is asked"
}
a, ask, err := DecodeAsk(e.Key, current)
if err != nil {
return nil, err.Error()
}
if ask.ID != e.Ask {
return nil, "the key holds the asking " + ask.ID + " now, not the one that ended"
}
if k.Kind == asks.AskKindRecord && put != nil && put.Ask == e.Key && PutAsking(*put) == a.Asking {
return nil, "a put asked it, and its put is refused when it lapses (ADR 0546)"
}
asking := max(AskingSeconds(now), a.Asking+1)
ask.ID, ask.Expires = meshdeliveryclient.AskingID(e.Key, asking), occ.FormatTime(now.Add(AskLasts))
value, err := EncodeAsk(e.Key, assignerAsk(asking, ask))
if err != nil {
return nil, err.Error()
}
return value, ""
}
// ChannelAsksBound is the most bytes of asks the assigner asks the operator-channel seat's verb asks for at its consumer's
// bind: every ended asking of its held asks fits.
const ChannelAsksBound = 1 << 20
// ChannelAsksQuery is the arguments of the operator-channel seat's verb asks that answer, at the assigner's consumer's bind,
// every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks
// opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false
// when no held ask reads, and then nothing is asked.
func ChannelAsksQuery(held map[string][]byte) (query oc.AsksInput, ok bool) {
var since time.Time
for key, value := range held {
a, ask, err := DecodeAsk(key, value)
if err != nil {
continue
}
opened := ExpiresAt(ask).Add(-AskLasts)
if a.Asking >= 2 {
if at := time.Unix(int64(a.Asking), 0); at.Before(opened) {
opened = at
}
}
if !ok || opened.Before(since) {
since, ok = opened, true
}
}
if !ok {
return oc.AsksInput{}, false
}
return oc.AsksInput{State: "ended", Since: since.Add(-time.Minute).UTC().Format(time.RFC3339), Bound: ChannelAsksBound}, true
}
// EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there
// expired or cancelled, as the teller says it in its event ask-ended, so the assigner writes it again by the rules of that
// event (AskedAgain), with its compare-and-set: an ended ask said before the assigner's consumer existed is not delivered
// to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read
// is the error.
func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) {
var got oc.AsksAnswer // the seat's declared answer: each ask's key and its state, and the cut
if err := json.Unmarshal(answer, &got); err != nil {
return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err)
}
byID := map[string]string{}
for key, value := range held {
if _, ask, err := DecodeAsk(key, value); err == nil {
byID[ask.ID] = key
}
}
for _, v := range got.Asks {
if v.State != occ.OutcomeExpired && v.State != occ.OutcomeCancelled {
continue
}
_, id, ok := oc.ParseAsksByAskerIDKey(v.Key)
if key, held := byID[id]; ok && held {
ended = append(ended, tellerevents.Ended{Key: key, Ask: id, Outcome: v.State})
}
}
slices.SortFunc(ended, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
return ended, got.Cut, nil
}
@@ -3,15 +3,12 @@ package assign
import (
"errors"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
@@ -22,66 +19,15 @@ import (
// assigner no longer says it straight (never both).
// FindingsBucket is the assigner's findings.
// FindingsState is the assigner's state of its findings, and FindingsBucket its bucket.
const FindingsState = "findings"
var FindingsBucket = health.FindingsBucket(delivery.AssignerModule)
func FindingsBucket() string { return assignment.Module() + "_" + FindingsState }
// KeyID is an id as the parts of a key: each dot-separated part made safe for a bus key, as the key of a condition is
// (mcc.Key), so a finding's key keeps the shape its condition had.
func KeyID(id string) string {
key := mcc.Key("x", id, "x")
return strings.TrimSuffix(strings.TrimPrefix(key, "x."), ".x")
}
// Observed is what the assigner observes wrong, before it is the seat mesh-conditions' generated finding.
type Observed struct {
Scope, ID, Kind, Machine string
Also []string
Severity string
Summary, Said string
Headline, Explanation string
Needs, Source string
Actions []mcseat.Action
}
// Finding is o as the seat's generated finding at at (design 47 G §686, A-22): its id made safe as a key's part, the resolved
// words not a field of it, its observer the assigner; the writer completes its `since`.
func (o Observed) Finding(at time.Time) mcseat.Finding {
at = at.UTC()
id := KeyID(o.ID)
said := o.Said
if said == "" {
said = o.Summary
}
sev := o.Severity
if sev == "" {
sev = mcseat.FindingSeverityWarning
}
return mcseat.Finding{Scope: o.Scope, ID: id, Kind: o.Kind, Severity: sev, Summary: o.Summary, Headline: o.Headline,
Explanation: o.Explanation, Needs: o.Needs, Actions: o.Actions, Source: o.Source, Observer: assignment.Module(), At: at,
Evidence: []mcseat.Evidence{{At: at, Said: said}},
Subject: mcseat.Subject{Scope: o.Scope, ID: id, Machine: o.Machine, Also: o.Also}}
}
// CallUnanswered is the observation of a kept call to callee (a seat verb's or a tool's key) about id (what the assigner waits
// on: a node, a key), unanswered since since, its last fault said. Its key is `call.<id>.call-unanswered`.
func CallUnanswered(id, callee, machine, fault, source string, since time.Time) Observed {
return Observed{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Machine: machine, Severity: mcseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the call %s about %s is unanswered since %s", callee, id, since.UTC().Format(time.RFC3339)),
Said: fault, Headline: "A call whose answer is kept is not answered",
Explanation: fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has "+
"not been answered since %s. Its last fault: %s.", callee, since.UTC().Format(time.RFC3339), fault),
Source: source}
}
// FindingOf is a said condition as a finding: from the condition's own key `mesh-delivery-assigner.<kind>.<id…>` (the
// subject's id when the key names none), its words unchanged but the resolved ones. raised is false for a clear, whose
// finding names only its scope, id and kind.
func FindingOf(c oc.Condition, at time.Time) (f mcseat.Finding, raised bool, err error) {
rest, ok := strings.CutPrefix(c.Key, assignment.Module()+".")
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
// a clear, whose key is to be deleted; value is nil then.
func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".")
if !ok || rest == "" {
return f, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
}
kind, id, _ := strings.Cut(rest, ".")
if id == "" {
@@ -89,30 +35,33 @@ func FindingOf(c oc.Condition, at time.Time) (f mcseat.Finding, raised bool, err
}
scope := c.Subject.Scope
if scope == "" {
scope = mcc.ScopeModule
scope = health.ScopeModule
}
if c.State == occ.StateCleared {
return mcseat.Finding{Scope: scope, ID: KeyID(id), Kind: kind}, false, nil
key = health.FindingKey(scope, id, kind)
if c.State == asks.StateCleared {
return key, nil, false, nil
}
if c.State != occ.StateRaised {
return f, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
if c.State != asks.StateRaised {
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
}
summary := c.Summary
if summary == "" {
summary = c.Headline
}
var actions []mcseat.Action
var actions []health.Action
for _, a := range c.Actions {
actions = append(actions, mcseat.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
}
return Observed{Scope: scope, ID: id, Kind: kind, Machine: c.Subject.Machine, Severity: string(c.Severity), Summary: summary,
Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs, Actions: actions, Source: "assigner." + kind}.Finding(at), true, nil
value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine},
Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved,
Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind,
Observer: delivery.AssignerModule, At: at.UTC()})
return key, value, true, err
}
// Findings is where the assigner's findings go: its finding writer's write and clear (design 47 G §689, A-29), by the
// finding's key `<scope>.<id>.<kind>`.
// Findings is where the assigner's findings go: the bucket's put and delete.
type Findings struct {
Put func(f mcseat.Finding) error
Put func(key string, value []byte) error
Delete func(key string) error
Keys func() ([]string, error)
Now func() time.Time
@@ -121,21 +70,20 @@ type Findings struct {
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
// findings bucket at all. A finding that cannot be written is the error.
func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
return func(c oc.Condition) error {
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
return func(c asks.Condition) error {
var errs []error
if f.Put != nil {
now := time.Now
if f.Now != nil {
now = f.Now
}
finding, raised, err := FindingOf(c, now())
key := views.FindingKey(finding.Scope, finding.ID, finding.Kind)
key, value, raised, err := FindingOf(c, now())
switch {
case err != nil:
errs = append(errs, err)
case raised:
if err := f.Put(finding); err != nil {
if err := f.Put(key, value); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
}
default:
@@ -154,18 +102,3 @@ func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
return errors.Join(errs...)
}
}
// KindActingCycle is the finding of a want held on a cycle of acting waits no rule breaks (design 47 G A-323, A-330): scope module,
// id `<node>.<module>` of the held want, naming the cycle with each edge's step; cleared by the next pass that finds it held on no
// cycle.
const KindActingCycle = "acting-cycle"
// ActingCycleFinding is the finding acting-cycle of module's want on node, held on cycle.
func ActingCycleFinding(node, module, cycle string) Observed {
return Observed{Scope: mcc.ScopeModule, ID: node + "." + module, Kind: KindActingCycle, Machine: node, Severity: "warning",
Summary: fmt.Sprintf("the want of %s on %s is held on a cycle of acting waits: %s", module, node, cycle),
Headline: fmt.Sprintf("%s on %s waits on a cycle of acting parts", module, node),
Explanation: fmt.Sprintf("Each acting part on the cycle %s waits for another's new version to run, so none moves by itself. "+
"assigner_release lets one move with the version running.", cycle),
Source: "assigner." + KindActingCycle}
}
@@ -1,6 +0,0 @@
package assign
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
@@ -0,0 +1,22 @@
package assign
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) asks.Condition {
key := delivery.AssignerModule + ".seat-states"
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
if len(unbound) == 0 {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -3,17 +3,15 @@ package assign
import (
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
@@ -21,7 +19,7 @@ import (
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
// PoliciesBucket is the assigner's state `policies`.
func PoliciesBucket() string { return assignment.Module() + "_" + buckets.PoliciesState }
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
@@ -62,22 +60,21 @@ const KindProvisionFailed = "provision-failed"
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
func ProvisionFailedKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionFailed)
return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed)
}
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) {
func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Source: "assigner." + KindProvisionFailed}, at)
return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed,
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)},
Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}},
Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()})
}
// encode is the observation as the assigner's finding at at.
func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil }
func also(provider, consumer string) []string {
if provider == "" || provider == consumer {
return nil
@@ -90,40 +87,44 @@ const KindProvisionMissing = "provision-missing"
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
func ProvisionMissingKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionMissing)
return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
}
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) {
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
parts := strings.SplitN(key, ".", 3)
if len(parts) != 3 {
return mcseat.Finding{}, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
}
node, module, provision := parts[0], parts[1], parts[2]
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
"no provider of it may be placed", module, node, provision, bound, module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Source: "assigner." + KindProvisionMissing}, at)
return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
}
// KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6).
// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6).
const KindUnplaced = "unplaced"
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
func UnplacedKey(node string) string {
return mcc.Key(mcc.ScopeMachine, node, KindUnplaced)
return health.FindingKey(health.ScopeMachine, node, KindUnplaced)
}
// UnplacedFinding is the finding of a node no assignments names since the lease write.
func UnplacedFinding(node string, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s runs, but no assignments names it: since the controller handed its lease away nothing composes it, "+
// UnplacedFinding is the finding of a node no placement names since the lease write.
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+
"so the assigner writes it no want", node)
return encode(Observed{Scope: mcc.ScopeMachine, ID: node, Kind: KindUnplaced, Machine: node, Summary: summary,
Headline: clipHeadline(node + " has no assignments"), Needs: "place it with mesh-module-assigner.propose.",
Explanation: occ.NeedsYou + "place it with mesh-module-assigner.propose. " + summary + ".",
Source: "assigner." + KindUnplaced}, at)
return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced,
Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary,
Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.",
Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".",
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
}
func clipHeadline(s string) string {
@@ -2,16 +2,10 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"mesh-delivery-assigner/internal/puts"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
@@ -25,7 +19,7 @@ const ComposeWithin = 10 * time.Minute
// PutWorld is what one step of a put reads.
type PutWorld struct {
Node, Module string
Put puts.Put
Put delivery.Put
// Assigned says the module stands assigned to the node.
Assigned bool
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
@@ -36,7 +30,7 @@ type PutWorld struct {
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *ComposedDigest
Composed *delivery.ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
@@ -48,16 +42,13 @@ type PutWorld struct {
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *nodeengine.HaveByNodeModuleValue
// Running is the node-engine seat's last `running` of the module on the node; nil when it said none (design 47 G §309). A put
// ends on it, never on the have.
Running *nodeengine.Running
Now time.Time
Have *delivery.Have
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
type PutNext struct {
Put puts.Put
Put delivery.Put
// Ask writes the put's ask about the move (PutAsk).
Ask bool
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
@@ -69,7 +60,7 @@ type PutNext struct {
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
func StepPut(w PutWorld) PutNext {
p := w.Put
if puts.Ended(p.Step) {
if delivery.PutEnded(p.Step) {
return PutNext{Put: p}
}
say := func(step, format string, args ...any) PutNext {
@@ -77,21 +68,21 @@ func StepPut(w PutWorld) PutNext {
}
fp := short(p.FP)
if !w.Assigned {
return say(puts.Refused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its assignments first)", w.Module, w.Node)
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
}
switch w.Build {
case builds.BuildDone:
case builds.BuildFailed:
return say(puts.Refused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case delivery.BuildDone:
case delivery.BuildFailed:
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case "":
if p.FP != w.Newest {
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
}
return say(puts.Build, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
default:
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
}
if w.Composed == nil || !w.Granted {
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
@@ -99,33 +90,33 @@ func StepPut(w PutWorld) PutNext {
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
}
since := w.Now
if p.Step == puts.Compose && len(p.Steps) > 0 {
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
since = p.Steps[len(p.Steps)-1].At
}
if w.Now.Sub(since) > ComposeWithin {
return say(puts.Refused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
}
return say(puts.Compose, "waits up to %s: %s yet", ComposeWithin, missing)
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
}
d := *w.Composed
if p.Step == puts.Wanted || p.Step == puts.Holding {
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
return PutNext{Put: applied(w, p)}
}
p.Composed = d.Digest
if needsPerson(w, d) {
switch w.Answer {
case OptionNo:
return say(puts.Refused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
case OptionYes:
default:
if w.AskExpired {
return say(puts.Refused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
}
next := say(puts.Person, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
if p.Ask == "" {
next.Ask = true
next.Put.Ask, _ = asks.RecordAskKey(w.Node, w.Module, d.Digest)
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
}
return next
}
@@ -134,11 +125,11 @@ func StepPut(w PutWorld) PutNext {
case w.Snapshot != "":
p.Snapshot = w.Snapshot
case w.SnapshotFailed != "":
return say(puts.Refused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
case w.Snapshotting:
return say(puts.Snapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
default:
next := say(puts.Snapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next.Snapshot = true
return next
}
@@ -151,23 +142,23 @@ func StepPut(w PutWorld) PutNext {
if p.Snapshot != "" {
words += "; the streams were snapshotted: " + p.Snapshot
}
next := PutNext{Put: applied(w, p.Said(w.Now, puts.Wanted, words))}
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
next.Warranted = w.Bus && p.Ask != ""
return next
}
// applied is a wanted put once the node says how it went: holding or done when the node-engine seat's `running` names the put's
// fingerprint (design 47, *Compositions, wants and the apply*: from G a put ends on `running.<node>.<module>` naming its
// fingerprint, never on the have). A `running` without a fingerprint, or one that says the module stopped, ends none. The have
// still says a failed health check in the put's words.
func applied(w PutWorld, p puts.Put) puts.Put {
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP {
if p.Hold {
return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
}
return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
return p
}
if h := w.Have; h != nil && h.Fp == p.FP && h.Composed == p.Composed && failed(*h) {
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
return p
@@ -176,8 +167,8 @@ func applied(w PutWorld, p puts.Put) puts.Put {
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d ComposedDigest) bool {
runs := w.Have != nil && w.Have.Fp == w.Put.FP
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
@@ -187,22 +178,22 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
if !w.Recorded {
return false
}
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
func PutWant(p puts.Put) mwc.Want {
return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
func PutWant(p delivery.Put) delivery.Want {
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
func PutOwns(p puts.Put) (want *mwc.Want, owns bool) {
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
switch p.Step {
case puts.Wanted, puts.Holding:
case delivery.PutWanted, delivery.PutHolding:
w := PutWant(p)
return &w, true
case puts.Done, puts.Released, puts.Refused:
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
return nil, false
}
return nil, true
@@ -211,7 +202,7 @@ func PutOwns(p puts.Put) (want *mwc.Want, owns bool) {
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
func PutAsking(p puts.Put) int {
func PutAsking(p delivery.Put) int {
if n := p.At.UnixMilli(); n > 0 {
return int(n)
}
@@ -219,43 +210,43 @@ func PutAsking(p puts.Put) int {
}
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
func PutAct(node, module, fp, composed string) occ.Act {
return occ.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
func PutAct(node, module, fp, composed string) asks.Act {
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
}
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p puts.Put, d ComposedDigest, bus bool, now time.Time) (asks.Ask, error) {
yes := oc.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: occ.Acknowledge}
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + d.Words,
JudgedNeither: "recreates no container and moves no data",
JudgedUnchanged: "changes nothing the node runs",
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := occ.FormatTime(now.Add(7 * 24 * time.Hour))
expires := now.Add(7 * 24 * time.Hour)
if bus {
binds, err := occ.ActDigest(PutAct(node, module, p.FP, d.Digest))
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
if err != nil {
return asks.Ask{}, err
return delivery.AssignerAsk{}, err
}
yes.Level, yes.Binds = occ.Approve, binds
yes.Level, yes.Binds = asks.Approve, binds
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
expires = occ.FormatTime(now.Add(occ.ApproveLasts))
expires = now.Add(asks.ApproveLasts)
}
return assignerAsk(PutAsking(p), oc.Ask{
ID: meshdeliveryclient.AskingID(key, PutAsking(p)),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, PutAsking(p)),
Headline: fmt.Sprintf("Put %s on %s?", module, node),
Explanation: explanation,
Options: []oc.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: occ.Acknowledge}},
Who: occ.Operator,
Options: []asks.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
Who: asks.Operator,
Expires: expires,
OnExpiry: "the node keeps what it runs",
About: key,
}), nil
}, Asking: PutAsking(p)}, nil
}
@@ -7,48 +7,37 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assignment"
plc "mesh-delivery-assigner/internal/assignment"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The seat mesh-module-assigner (novox/hq ADR 0567), held by the assigner beside its wants: assignments is its state
// `assignments`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current assignments and the
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
// digest.
type ProposeIn struct {
ID, By, Why string
At time.Time
Changes []mma.ProposalChange
Current map[string]mma.Assignments
Changes []delivery.PlacementProposalChange
Current map[string]delivery.Placement
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
Warrant *mma.Warrant
// SeatListRead says the host has read the seat list from the seat mesh-catalog (ADR 0636 §1); until then a mesh seat
// the bootstrapped definitions do not name is not judged held once (plc.CheckAssignments).
SeatListRead bool
Warrant *delivery.AddNodeWarrant
}
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
// the laid assignments of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
// CheckAssignments refuses of the result laid over every assignments (two modules on one host port, a provider not placed,
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string) {
p := mma.Proposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]int64{}, State: mmac.ProposalAnswering}
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
@@ -60,39 +49,39 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string)
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if c.Op == mmac.OpSeed {
if c.Op == delivery.OpSeed {
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
}
}
laid, lay := plc.LayChanges(in.Current, in.Changes)
laid, lay := delivery.LayChanges(in.Current, in.Changes)
findings = append(findings, lay...)
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
all := map[string]mma.Assignments{}
all := map[string]delivery.Placement{}
for node, pl := range in.Current {
all[node] = pl
}
out := map[string]*mma.Assignments{}
out := map[string]*delivery.Placement{}
for node, pl := range laid {
if before, ok := in.Current[node]; ok && mmac.SameAssignments(before, pl) {
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
continue
}
pl := pl
all[node] = pl
out[node] = &pl
p.Base[node] = int64(in.Revisions[node])
p.Base[node] = in.Revisions[node]
}
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
// candidate is no assignments, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
for _, node := range plc.RemovedNodes(p) {
// candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
for _, node := range p.RemovedNodes() {
if _, ok := in.Current[node]; !ok {
continue
}
delete(all, node)
out[node] = nil
p.Base[node] = int64(in.Revisions[node])
p.Base[node] = in.Revisions[node]
}
if len(findings) == 0 && len(out) == 0 {
findings = append(findings, "the changes leave every assignments as it is")
findings = append(findings, "the changes leave every placement as it is")
}
held, recorded := in.Held, in.Recorded
if held == nil {
@@ -105,7 +94,7 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string)
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
joining := map[string]bool{}
for _, c := range in.Changes {
if c.Op == mmac.OpAddNode {
if c.Op == delivery.OpAddNode {
joining[c.Node] = true
}
}
@@ -113,7 +102,7 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string)
was := recorded
recorded = func(node string) bool { return joining[node] || was(node) }
}
for _, r := range plc.CheckAssignments(all, held, recorded, in.SeatListRead) {
for _, r := range delivery.CheckPlacements(all, held, recorded) {
findings = append(findings, r)
}
return p, out, findings
@@ -122,19 +111,19 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string)
// ControllerModule is the controller's module, which nothing places again (issue 1126).
const ControllerModule = "mesh-controller"
// ControllerEnded is why assignments of the controller is refused: the parts that acted only once its lease was handed
// ControllerEnded is why a placement of the controller is refused: the parts that acted only once its lease was handed
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
// would write beside them, silently.
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
// whole assignments, or in what the changes lay. None when nothing does.
func ControllerRefusals(changes []mma.ProposalChange, laid map[string]mma.Assignments) []string {
// whole placement, or in what the changes lay. None when nothing does.
func ControllerRefusals(changes []delivery.PlacementProposalChange, laid map[string]delivery.Placement) []string {
var out []string
nodes := map[string]bool{}
for i, c := range changes {
places := c.Module == ControllerModule && c.Op != mmac.OpTakeOff
if c.Whole != nil && slices.Contains(mmac.Assigned(*c.Whole), ControllerModule) {
places := c.Module == ControllerModule && c.Op != delivery.OpTakeOff
if c.Whole != nil && slices.Contains(c.Whole.Assigned(), ControllerModule) {
places = true
}
if places {
@@ -143,7 +132,7 @@ func ControllerRefusals(changes []mma.ProposalChange, laid map[string]mma.Assign
}
}
for _, node := range slices.Sorted(maps.Keys(laid)) {
if !nodes[node] && slices.Contains(mmac.Assigned(laid[node]), ControllerModule) {
if !nodes[node] && slices.Contains(laid[node].Assigned(), ControllerModule) {
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
}
}
@@ -151,23 +140,23 @@ func ControllerRefusals(changes []mma.ProposalChange, laid map[string]mma.Assign
}
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
// mesh-sdk mmac.Keys.Writes), the revision of the node's assignments it is written at (0: it has none, so
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
// it is created) and its value (nil: it is deleted).
type Write struct {
Node, Key string
Base uint64
Value *mma.Assignments
Value *delivery.Placement
}
// Writes is every key a proposal writes, by node order.
func Writes(p mma.Proposal, laid map[string]*mma.Assignments) ([]Write, error) {
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
var out []Write
for _, node := range plc.Nodes(p) {
key, err := mmac.NodeKey(node)
for _, node := range p.Nodes() {
key, err := delivery.PlacementNodeKey(node)
if err != nil {
return nil, err
}
out = append(out, Write{Node: node, Key: key, Base: uint64(p.Base[node]), Value: laid[node]})
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
}
return out, nil
}
@@ -181,56 +170,56 @@ const (
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
// module, or node and provision for a pin).
func CallClearedCondition(kind, node, module string) oc.Condition {
return oc.Condition{Key: assignment.Module() + "." + kind + "." + node + "." + module,
Subject: oc.ConditionSubject{Scope: "module", ID: module, Machine: node}, State: occ.StateCleared,
Resolved: "the assigner calls no verb of the controller any more: assignments is the one source."}
func CallClearedCondition(kind, node, module string) asks.Condition {
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
}
// AssignmentsAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
func AssignmentsAct(p mma.Proposal) occ.Act {
return occ.Act{"assignments": "write", "proposal": p.ID, "changes": plc.ChangesDigest(p)}
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
func PlacementAct(p delivery.PlacementProposal) asks.Act {
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
}
// AssignmentsAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
// another asking, under an id of its own.
func AssignmentsAsking(p mma.Proposal, at time.Time) int {
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
return int(at.Unix())
}
// AssignmentsAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
func AssignmentsAsk(key string, p mma.Proposal, brief string, asking int, now time.Time) (asks.Ask, error) {
binds, err := occ.ActDigest(AssignmentsAct(p))
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
binds, err := asks.ActDigest(PlacementAct(p))
if err != nil {
return asks.Ask{}, err
return delivery.AssignerAsk{}, err
}
nodes := plc.Nodes(p)
headline := fmt.Sprintf("Change assignments on %s?", strings.Join(nodes, ", "))
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = fmt.Sprintf("Change assignments on %d machines?", len(nodes))
nodes := p.Nodes()
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
}
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: headline,
Explanation: brief,
Options: []oc.Option{
{ID: OptionYes, Label: "Approve", Does: "the assignments of " + strings.Join(nodes, ", ") + " is written as proposed",
Level: occ.Approve, Binds: binds},
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
Level: asks.Approve, Binds: binds},
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(occ.ApproveLasts)),
Who: asks.Operator,
Expires: now.Add(asks.ApproveLasts),
OnExpiry: "nothing is written, and the proposal ends",
About: key,
}), nil
}, Asking: asking}, nil
}
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
func Rest(p mma.Proposal, written map[string]*mma.Assignments) (mma.Proposal, bool) {
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
rest := p
rest.Changes, rest.Base = nil, map[string]int64{}
rest.Changes, rest.Base = nil, map[string]uint64{}
for _, c := range p.Changes {
if _, done := written[c.Node]; !done {
rest.Changes = append(rest.Changes, c)
@@ -241,6 +230,6 @@ func Rest(p mma.Proposal, written map[string]*mma.Assignments) (mma.Proposal, bo
rest.Base[node] = base
}
}
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", mmac.ProposalAnswering
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
return rest, len(rest.Changes) > 0
}
@@ -2,71 +2,67 @@ package assign
import (
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/keys"
"slices"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
plc "mesh-delivery-assigner/internal/assignment"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestTheAssignmentsSeatDecides(t *testing.T) {
anchor := mma.Assignments{V: keys.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []mma.PlacedModule{{Module: "store", Ports: map[string]int64{"5432": 5432}}, {Module: "web"}}}
laptop := mma.Assignments{V: keys.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
in := func(changes ...mma.ProposalChange) ProposeIn {
func TestThePlacementSeatDecides(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]mma.Assignments{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Held: func(m string) bool { return m != "ghost" },
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
}
move := []mma.ProposalChange{{Op: mmac.OpTakeOff, Node: "anchor", Module: "web"}, {Op: mmac.OpPlace, Node: "laptop", Module: "web"}}
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
p, laid, findings := Propose(in(move...))
if len(findings) != 0 || p.State != mmac.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(mmac.Assigned(*laid["laptop"]), []string{"shell", "web"}) || !slices.Equal(mmac.Assigned(*laid["anchor"]), []string{"store"}) {
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
}
if _, err := plc.EncodeProposal(p); err != nil {
if _, err := delivery.EncodePlacementProposal(p); err != nil {
t.Fatalf("the proposal cannot be written: %v", err)
}
for name, c := range map[string]struct {
in ProposeIn
want string
}{
"a module not held": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "anchor", Module: "web"}), "leave every assignments as it is"},
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
// act without its lease.
"the controller placed again": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "anchor", Whole: &mma.Assignments{V: keys.Version, Node: "anchor",
Decided: anchor.Decided, Modules: []mma.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
"a pin of nobody placed": {in(mma.ProposalChange{Op: mmac.OpPin, Node: "laptop", Provision: "db", Provider: &mma.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
"the controller placed again": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "anchor", Whole: &delivery.Placement{V: delivery.Version, Node: "anchor",
Decided: anchor.Decided, Modules: []delivery.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
}
}
// Every kind of change is taken: the assignments is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
for _, c := range []mma.ProposalChange{
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: mmac.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int64{"80": 8080}, Bindings: map[string]mma.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: mmac.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: mmac.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
for _, c := range []delivery.PlacementProposalChange{
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
if _, _, f := Propose(in(c)); len(f) != 0 {
t.Errorf("%s is refused: %v", c.Op, f)
@@ -74,8 +70,8 @@ func TestTheAssignmentsSeatDecides(t *testing.T) {
}
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
if rest, ok := Rest(p, map[string]*mma.Assignments{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != mmac.ProposalAnswering || rest.Must != nil {
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
t.Errorf("the rest = %+v, %v", rest, ok)
}
if _, ok := Rest(p, laid); ok {
@@ -83,10 +79,10 @@ func TestTheAssignmentsSeatDecides(t *testing.T) {
}
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
for _, unbound := range [][]string{{plc.PulsesBucket()}, nil} {
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
c := SeatWaitsCondition(unbound)
if _, err := occ.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == occ.StateRaised) ||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+plc.PulsesBucket())) {
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
}
}
@@ -95,13 +91,13 @@ func TestTheAssignmentsSeatDecides(t *testing.T) {
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
t.Fatalf("writes = %+v, %v", writes, err)
}
a, err := AssignmentsAsk("assignments.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || occ.Check(mustAsk(t, a), time.Unix(100, 0)) != nil || mustAsk(t, a).Options[0].Level != occ.Approve || occ.Performs(mustAsk(t, a).Options[0], AssignmentsAct(p)) != nil {
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
t.Fatalf("the ask = %+v, %v", a, err)
}
q := p
q.Changes = append(slices.Clone(p.Changes), mma.ProposalChange{Op: mmac.OpUnpin, Node: "laptop", Provision: "x"})
if occ.Performs(mustAsk(t, a).Options[0], AssignmentsAct(q)) == nil {
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
}
@@ -3,7 +3,7 @@ package assign
import (
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
@@ -2,39 +2,28 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"maps"
"mesh-delivery-assigner/internal/puts"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
Name string
FP string
Built bool
// BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole
// (design 47 G §575, A-162): the words of a hold under rule 1.
BuildFailed bool
BuildError string
Held bool
Granted bool
Name string
FP string
Built bool
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
@@ -45,7 +34,7 @@ type Module struct {
Manifest catalogue.Manifest
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
Provisions func(node string) map[string]meshprovisionerclient.Provision
Provisions func(node string) map[string]provisions.Provision
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
Answering func(provision string) []catalogue.Manifest
@@ -69,10 +58,14 @@ type Module struct {
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (ComposedDigest, bool)
Composed func(node, fp string) (delivery.ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
// approved release is known to carry it. Nil: none is read.
Release func(fp string) (string, bool)
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
@@ -83,88 +76,21 @@ type Module struct {
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]puts.Put
Puts map[string]delivery.Put
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
Needs []string
// FormNeeds are the reference forms the module's build at FP is written in (`form:<name>`, the builder's build key's
// needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the
// build names none, and holds nothing.
FormNeeds []string
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). It is the one
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the assignments (issue 1126). Nil: no node does.
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
Self func(node string) bool
// KeptBuilds answers what a want at a fingerprint names of its build: its parts and its keeping node as the builder's `build.ended`
// names them, or the released version and parts of the catalog's `version-released` (design 47 G §379, §526, A-96), for a node that states the capability WantParts. Nil: none is read.
KeptBuilds func(fp string) (KeptBuild, bool)
// MachineHeld says whether the node's own network condition is open, which holds the machine as a whole: no want moves to it
// until that clears (design 47 G §706 *M's own network*). Nil: none is held.
MachineHeld func(node string) bool
// Acting are the acting edges of the assignment graph entry of this module on node that the assigner holds (composing, placing)
// and whose wait was not released by hand (design 47 G §266, §268): the want moves only once each acting part's version runs
// on the edge's node, as Nodes.Running says it. Nil: none.
Acting func(node string) []ActingEdge
}
// ActingEdge is one acting edge: Part's version Fp acts in Step on Node, before the entry moves.
type ActingEdge struct{ Node, Part, Fp, Step string }
// Runs says the part's version of the edge runs on its node, as the node-engine seat's `running` says it.
func (n Nodes) Runs(e ActingEdge) bool {
if n.Running != nil {
r, ok := n.Running[e.Node][e.Part]
return ok && !r.Stopped && r.Fingerprint == e.Fp
}
h, ok := n.Haves[e.Node][e.Part]
return ok && healthy(h, e.Fp)
}
// actingWaiting says why the module's want on node waits for an acting part's new version (§266): the first edge not run.
func (m Module) actingWaiting(node string, n Nodes) (string, bool) {
if m.Acting == nil {
return "", false
}
for _, e := range m.Acting(node) {
if !n.Runs(e) {
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", m.Name, node, e.Step, e.Part, short(e.Fp), e.Node), true
}
}
return "", false
}
// WantParts is the capability a node-engine of G states (ADR 0614 §4, design 47 G §525): it reads a want's `parts` and `kept-on`.
const WantParts = "want-parts"
// KeptBuild is what a want names of a build: the digest of each of its parts by `<module>:<artifact>`, and, as the builder's
// `build.ended` says it, the build node that keeps a kept build; for a trunk build the released version the catalog's
// `version-released` says (design 47 G §379, §526, A-96).
type KeptBuild struct {
Parts map[string]string
KeptOn string
Version string
}
// withKept is w naming the kept build at its fingerprint, the parts by digest and the keeping node, only while the node's
// engine states WantParts (design 47 G §525, §526); a want of a build that is not kept, or of a node that does not state it,
// is w.
func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want {
if m.KeptBuilds == nil {
return w
}
k, ok := m.KeptBuilds(w.Fp)
if !ok || !slices.Contains(n.Reads[node], WantParts) {
return w
}
w.Parts, w.KeptOn, w.Version = k.Parts, k.KeptOn, k.Version
return w
}
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
return m.Composed(node, fp)
}
@@ -172,26 +98,19 @@ func (m Module) composition(node, fp string) (ComposedDigest, bool) {
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy
// while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider
// before its consumers*). Nil: the seat's events are not heard, and the have says it.
Running map[string]map[string]nodeengine.Running
Haves map[string]map[string]delivery.Have
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
func healthy(h delivery.Have, fp string) bool {
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}
// Failed says whether a have's health check failed once the node is done moving.
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
// name, ADR 0528 clause 3). Empty when none is online.
func Canary(nodes []string, online map[string]bool) string {
@@ -218,65 +137,50 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) {
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
if m.BuildFailed {
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError)
}
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP))
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
case m.Held:
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
case m.MachineHeld != nil && m.MachineHeld(node):
return false, mwc.RuleNotHeld, fmt.Sprintf("the condition machine.%s.network is open: no want moves to %s until its own network is healthy", node, node)
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
}
if !m.Granted {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if words, lacking := m.readerLacking(node, n); lacking {
return false, mwc.RuleComposed, words
return false, delivery.RuleComposed, words
}
if _, ok := m.composition(node, m.FP); !ok {
return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.actingWaiting(node, n); waiting {
return false, mwc.RuleComposed, words
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, mwc.RuleProvisions, words
return false, delivery.RuleProvisions, words
}
if words, waiting := m.verbWaiting(node); waiting {
return false, mwc.RuleProvisions, words
return false, delivery.RuleProvisions, words
}
if words, waiting := m.busStateWaiting(node); waiting {
return false, mwc.RuleProvisions, words
return false, delivery.RuleProvisions, words
}
return true, 0, ""
}
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
// manifest needs that the engine does not state, read as the baseline when it states none, a bootstrapped seat counting as
// read by every engine (graph.MissingReads, ADR 0636 §2). It is held under rule 7 (the
// manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
//
// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a
// first form, so a build written in one is never held by its forms; the words name the forms it reads none of.
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
missing := graph.MissingReads(m.Needs, n.Reads[node])
if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) {
missing = slices.Clone(m.FormNeeds)
}
missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node])
if len(missing) == 0 {
return "", false
}
@@ -322,7 +226,7 @@ func (m Module) provisionWaiting(node string) (string, bool) {
}
}
for _, key := range slices.Sorted(maps.Keys(keyed)) {
if p := keyed[key]; p.State != meshprovisionerclient.Provided && slices.Contains(needed, p.Provision) {
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
if p.Why != "" {
words += ": " + p.Why
@@ -355,21 +259,6 @@ func (m Module) MissingProvisions(node string) []string {
return out
}
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
func (n Nodes) runsHealthy(node, module string) bool {
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy)
}
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
// module of which nothing was heard counts as not healthy. The have is never read for it.
func (n Nodes) RunsHealthy(node, module, fp string) bool {
r, ok := n.Running[node][module]
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
@@ -389,7 +278,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if at != "" && n.runsHealthy(at, p) {
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
return true, ""
}
}
@@ -420,9 +309,9 @@ func keepsData(m catalogue.Manifest) bool {
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
type Outcome struct {
Wants map[string]mwc.Want
Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *mwc.Held
Held *delivery.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
@@ -451,7 +340,7 @@ type Outcome struct {
type RecordNeed struct {
Node string
Digest string
Judgement Judgement
Judgement delivery.Judgement
Words string
}
@@ -472,8 +361,8 @@ func (h Holding) Reason() string {
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
@@ -481,7 +370,7 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.Fp != m.FP || !failed(h) {
if !ok || h.FP != m.FP || !failed(h) {
continue
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
@@ -496,7 +385,7 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)}
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
@@ -508,24 +397,24 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
break
}
before, had := wants[node]
var next mwc.Want
var next delivery.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: puts.Holding}, true
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
}
if put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp {
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
w.Composed = ul.Composed
}
if !had || !mwc.SameWant(normal(before), normal(w)) {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
}
@@ -536,29 +425,29 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !mwc.SameWant(normal(before), normal(w)) {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary {
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
if older {
_, composed := m.composition(node, have.Fp)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
}
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
switch {
case older:
ok, target = true, have.Fp
ok, target = true, have.FP
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
@@ -571,40 +460,40 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) &&
n.Haves[node][m.Name].Fp != m.FP {
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
next = n.withKept(m, node, mwc.Want{Fp: target, Composed: d.Digest})
next = delivery.Want{FP: target, Composed: d.Digest}
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, int64(rule), words
next.Newest, next.Rule, next.Words = m.FP, rule, words
at.FP = target
}
// A placed node's want names the build alone: it is the same target while the build is.
if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = mwc.Want{}, false
asked, hadLive = delivery.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
case held && had && before.Fp == m.FP:
case held && had && before.FP == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
@@ -622,7 +511,7 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words}
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
@@ -633,31 +522,31 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
case there && h.Phase != delivery.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
case had && (!there || h.Phase == delivery.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)}
if !mwc.SameWant(normal(before), normal(next)) {
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words}
if before.Fp == m.FP {
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = mwc.Want{Fp: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed {
next = delivery.Want{FP: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.Fp
at.FP = before.FP
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -670,26 +559,26 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
if !write {
continue
}
if !had || !mwc.SameWant(normal(before), normal(next)) {
if !had || !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
for node, before := range wants {
if !slices.Contains(m.Nodes, node) && !before.Removed {
out.Wants[node] = mwc.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
}
}
return out
}
// selfDerived is next with its Self derived from the assignments now (Module.Self), whichever rule made it, and whether it is
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
// assignments seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) {
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
switch {
case next.Removed:
return next, true
@@ -700,16 +589,16 @@ func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Wan
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.Fp)
d, there := m.composition(node, next.FP)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))}
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
return next, false
}
at := m
at.FP = next.Fp
at.FP = next.FP
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
@@ -729,8 +618,8 @@ func NoWantWords(rule int, words string) string {
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h nodeengine.HaveByNodeModuleValue) bool {
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
@@ -739,7 +628,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
out = append(out, other)
}
}
@@ -750,7 +639,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
return true
}
}
@@ -771,8 +660,8 @@ type NodeFailure struct {
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h nodeengine.HaveByNodeModuleValue) bool {
return h.Phase == nodeengineclient.PhaseRunning
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
@@ -784,10 +673,10 @@ func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
continue
}
if !n.RunsHealthy(node, m.Name, m.FP) {
if !healthy(h, m.FP) {
return false
}
ran++
@@ -797,7 +686,7 @@ func allRunHealthy(m Module, n Nodes) bool {
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
func HeldWords(h delivery.Have, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
@@ -814,13 +703,13 @@ func LiftedWords(module, fp string) string {
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w mwc.Want) mwc.Want {
func normal(w delivery.Want) delivery.Want {
w.V = 0
return w
}
func short(fp string) string {
part, err := catalogkey.KeyPart(fp)
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
@@ -831,18 +720,18 @@ func short(fp string) string {
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) {
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
@@ -854,13 +743,13 @@ func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running str
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d ComposedDigest) bool {
func (m Module) recorded(d delivery.ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
if d.Policy != "" {
return d.Policy == catalogue.PolicyRecord
return d.Policy == delivery.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
@@ -869,19 +758,19 @@ func (m Module) recorded(d ComposedDigest) bool {
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) {
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return mwc.Want{}, false
return delivery.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) {
return mwc.Want{}, false
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
return delivery.Want{}, false
}
d, ok := m.composition(node, running.Fp)
d, ok := m.composition(node, running.FP)
if !ok || !d.UserListOnly {
return mwc.Want{}, false
return delivery.Want{}, false
}
w := mwc.Want{Fp: running.Fp, Composed: d.Digest}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
@@ -890,6 +779,6 @@ func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g grantsstate.Grant, node, module string) bool {
func GrantNamesNode(g delivery.Grant, node, module string) bool {
return g.NamesNode(node, module)
}
File diff suppressed because it is too large Load Diff
@@ -1,138 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// A candidate's assignment graph (design 47 G §189, §190, A-201, A-235): computed per run on request from the candidate's catalog
// graph revision, kept apart from trunk's under the seat's key form candidate.<group>.<heads>, the group `run-<id>`, and dropped on
// the runner's group-ended of that run; it writes no want, and the gate's heads-replaced drops nothing.
// CandidateObject is the object name of a candidate's revision (the seat's key form candidate.<group>.<heads>).
func CandidateObject(group, heads string) string {
return meshwants.AssignmentRevisionsByCandidateGroupHeadsKey(group, heads)
}
// KeepCandidate keeps c as the candidate's revision of group and heads, numbered after the one kept of it, unless that one lists
// the same entries from the same inputs; its entry objects are written first, an entry trunk shares stored once.
func KeepCandidate(ctx context.Context, o Objects, c Computed, group, heads, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
if group == "" || heads == "" {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("a candidate's revision names its group and its heads, not %q and %q", group, heads)
}
was, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
has := err == nil
if err != nil && !errors.Is(err, ErrNoObject) {
return was, false, err
}
if has && Same(was, c.Revision) {
return was, false, nil
}
rev := c.Revision
rev.Kind, rev.Group, rev.Heads, rev.Cause = meshcatalogclient.KindCandidate, group, heads, cause
rev.Revision = was.Revision + 1
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return was, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return was, false, err
}
if err := o.Put(ctx, CandidateObject(group, heads), list); err != nil {
return was, false, fmt.Errorf("the candidate %s's revision in %s: %w", group, RevisionsObjects, err)
}
return rev, true, nil
}
// ReadCandidate is the candidate's revision kept of group, of heads when named, else the newest kept of the group; ErrNoCandidate
// when none is kept.
func ReadCandidate(ctx context.Context, o Objects, group, heads string) (meshwants.AssignmentRevision, error) {
if heads != "" {
rev, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
if errors.Is(err, ErrNoObject) {
err = fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads)
}
return rev, err
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return meshwants.AssignmentRevision{}, err
}
var newest meshwants.AssignmentRevision
found := false
for _, n := range names {
rev, err := readCandidateObject(ctx, o, n)
if err != nil {
return rev, err
}
if !found || rev.At != nil && newest.At != nil && rev.At.After(*newest.At) {
newest, found = rev, true
}
}
if !found {
return newest, fmt.Errorf("%w (group %q)", ErrNoCandidate, group)
}
return newest, nil
}
// CandidateNames is the object name of every revision kept of the candidate group; every candidate's when group is "".
func CandidateNames(ctx context.Context, o Objects, group string) ([]string, error) {
names, err := o.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var out []string
for _, n := range names {
g, _, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if ok && (group == "" || g == group) {
out = append(out, n)
}
}
return out, nil
}
// Remover is an object bucket that also removes an object.
type Remover interface {
Objects
// Delete removes the object of that name; one not held is no error.
Delete(ctx context.Context, name string) error
}
// DropCandidate removes every revision kept of the candidate group (its run's group-ended, A-235); the entries it alone listed are
// removed with the next revisions nothing pins. It says how many it removed.
func DropCandidate(ctx context.Context, o Remover, group string) (int, error) {
if strings.TrimSpace(group) == "" {
return 0, errors.New("a candidate to drop names its group")
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return 0, err
}
for i, n := range names {
if err := o.Delete(ctx, n); err != nil {
return i, fmt.Errorf("the candidate %s's revision %s: %w", group, n, err)
}
}
return len(names), nil
}
func readCandidateObject(ctx context.Context, o Objects, name string) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, name)
if err != nil {
return rev, err
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("the candidate's revision %s: %w", name, err)
}
return rev, nil
}
@@ -1,303 +0,0 @@
// Package assigngraph is the assigner's assignment graph (design 47, *Two graphs, one source of truth*; ADR 0656): the
// module versions placed on nodes, with the edges between versions, computed from the catalog graph of the branch each
// placement names, the placements and the nodes' capability readings, and kept per revision as objects of the seat
// mesh-wants' kept data assignment-revisions.
package assigngraph
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/assignment"
)
// The kinds of an edge of the assignment graph (ADR 0656; the seat's Edge).
const (
EdgeProvisioning = "provisioning"
EdgeContribution = "contribution"
EdgeActing = "acting"
)
// Inputs are what a revision is computed from, and nothing else (design 47 lines 1353-1356).
type Inputs struct {
// Halves is the catalog graph revision held of each watched branch, by branch.
Halves map[string]meshcatalogclient.Half
// Placements is every node's placement key, by node.
Placements map[string]mma.Assignments
// FirstBranch is the first watched branch of a module's registration, the branch a placement naming none runs
// (design 47 lines 522-523); "" when no registration is known for it.
FirstBranch func(module string) string
// Capabilities is each node's capability reading, by its digest.
Capabilities []meshwants.AssignmentRevisionCapabilitiesItem
// Definitions are the seat definitions read whole through the seat mesh-catalog's verb seat-definitions, by seat: of
// them, those an entry's catalog graph revision names in its definitions say which seat delivers a requirement (§708).
Definitions map[string]seatdef.Definition
}
// Computed is one assignment graph revision as computed, before it is numbered and kept.
type Computed struct {
Revision meshwants.AssignmentRevision
// Entries is every entry the revision lists, by its digest.
Entries map[string]meshwants.AssignmentEntry
// Uncomputed is each placed module whose entry could not be computed, as "<node>/<module>: why", sorted: it is said,
// never left out in silence.
Uncomputed []string
// Cycles are the cycles among the acting waits, within one step or across steps (Cycles, design 47 G §272, A-323): said, never
// left in silence.
Cycles []string
}
// placedAt is one placed module with what its entry is computed from.
type placedAt struct {
node string
placed mma.PlacedModule
branch string
entry meshwants.AssignmentEntry
langs map[string]bool
acts []actingRule
// provides is what its catalog entry provides; contributes the requirements it carries contributed values to (§709).
provides, contributes []string
// delivering is, by requirement, the seat that delivers it among the definitions its revision names (§708).
delivering map[string]seatdef.Definition
// optional says a requirement is one it runs without (ADR 0330 §3), by the entry's manifest (A-229).
optional func(string) bool
}
type actingRule struct{ seat, step, on, bound string }
// Compute is the assignment graph revision of in: one entry per module placed on a node whose branch's catalog graph
// holds it, its version the build the placement names or else the catalog entry's fingerprint (design 47 lines 909-914),
// with the edges of provisioning to the providers its placement binds or its node pins, else the provider on its node or
// the only provider (an entry held naming the candidates where none is), and of acting from each acting
// part's version to each version its rule names (design 47 *An acting part's new version acts first*: "the assigner
// gives an edge from the acting part's version to each version its rule names").
func Compute(in Inputs) Computed {
out := Computed{Entries: map[string]meshwants.AssignmentEntry{}}
var placed []*placedAt
branches := map[string]bool{}
for _, node := range slices.Sorted(maps.Keys(in.Placements)) {
mods := slices.Clone(in.Placements[node].Modules)
slices.SortFunc(mods, func(a, b mma.PlacedModule) int { return strings.Compare(a.Module, b.Module) })
for _, m := range mods {
branch := m.Branch
if branch == "" && in.FirstBranch != nil {
branch = in.FirstBranch(m.Module)
}
if branch == "" {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: no watched branch: its placement names none and no registration of it is known", node, m.Module))
continue
}
half, ok := in.Halves[branch]
if !ok {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph of the branch %s is not held", node, m.Module, branch))
continue
}
ce, ok := half.Entries[m.Module]
if !ok {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph revision %d of %s holds no entry of it", node, m.Module, half.Revision.Revision, branch))
continue
}
digest := ""
for _, ref := range half.Revision.Entries {
if ref.Module == m.Module {
digest = ref.Digest
}
}
if digest == "" {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph revision %d of %s lists no digest of its entry", node, m.Module, half.Revision.Revision, branch))
continue
}
fp := ce.Fp
if m.Build != "" {
fp = m.Build
}
p := &placedAt{node: node, placed: m, branch: branch, langs: map[string]bool{},
entry: meshwants.AssignmentEntry{V: keys.Version, Node: node, Module: m.Module, Branch: branch, Fp: fp, Catalog: digest}}
for lang := range ce.Toolchains {
p.langs[lang] = true
}
for _, r := range ce.Acts {
p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On, bound: r.Bound})
}
p.provides, p.contributes = ce.Provides, ce.Contributes
p.delivering = map[string]seatdef.Definition{}
for _, d := range half.Revision.Definitions {
if def, ok := in.Definitions[d.Seat]; ok && def.Delivers != "" {
if _, had := p.delivering[def.Delivers]; !had {
p.delivering[def.Delivers] = def
}
}
}
p.optional = func(string) bool { return false }
if man, err := manifestOf(ce); err == nil {
p.optional = man.IsOptional
}
placed = append(placed, p)
branches[branch] = true
}
}
// Each requirement points, by design 27's order (design 47 G §357, §708), to the named seat's holder, else a pin, else
// the provider on the consumer's node, else the only provider; otherwise the entry is held, naming the candidates, and
// nothing is guessed. A requirement contributes names gets a contribution edge to the same provider (§709).
holders := assignment.SeatHoldersOf(in.Placements)
for _, p := range placed {
ce := in.Halves[p.branch].Entries[p.placed.Module]
for _, r := range ce.Requires {
prov, held := resolve(p, r, placed, holders, in.Placements[p.node].Pins)
switch {
case held != nil:
p.entry.Held = append(p.entry.Held, *held)
case prov.Module != "":
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: prov.Node, Module: prov.Module, Provision: r})
if slices.Contains(p.contributes, r) {
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeContribution, Node: prov.Node, Module: prov.Module, Provision: r})
}
}
}
slices.SortFunc(p.entry.Held, func(a, b meshwants.Held) int { return strings.Compare(a.Requirement, b.Requirement) })
}
at := map[string]*placedAt{}
for _, p := range placed {
at[p.node+"/"+p.placed.Module] = p
}
for _, p := range placed {
for i, e := range p.entry.Edges {
if prov, ok := at[e.Node+"/"+e.Module]; ok {
p.entry.Edges[i].Fp = prov.entry.Fp
}
}
}
// Acting: an acting part on the entry's own node acts there (a node's seat is held on every node); else every node it
// is placed on (a seat held once for the mesh is held on one).
byModule := map[string][]*placedAt{}
for _, p := range placed {
byModule[p.placed.Module] = append(byModule[p.placed.Module], p)
}
for _, part := range slices.Sorted(maps.Keys(byModule)) {
holders := byModule[part]
for _, rule := range holders[0].acts {
for _, t := range placed {
if t.placed.Module == part || (rule.on != "" && rule.on != "every" && !t.langs[rule.on]) {
continue
}
from := holders
for _, h := range holders {
if h.node == t.node {
from = []*placedAt{h}
}
}
for _, h := range from {
t.entry.Edges = append(t.entry.Edges, meshwants.Edge{Kind: EdgeActing, Node: h.node, Module: part, Fp: h.entry.Fp,
Seat: rule.seat, Step: rule.step, Bound: rule.bound})
}
}
}
}
rev := meshwants.AssignmentRevision{V: keys.Version, Kind: meshcatalogclient.KindTrunk, Placements: assignment.ExportDigest(in.Placements),
Capabilities: in.Capabilities, Entries: []meshwants.NodeEntryRef{}}
for _, b := range slices.Sorted(maps.Keys(branches)) {
h := in.Halves[b]
rev.Catalogs = append(rev.Catalogs, meshwants.CatalogOf{Branch: b, Revision: h.Revision.Revision, Source: h.Revision.Source})
}
for _, p := range placed {
sortEdges(p.entry.Edges)
d := EntryDigest(p.entry)
out.Entries[d] = p.entry
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: p.node, Module: p.placed.Module, Digest: d})
}
out.Revision = rev
out.Cycles = Cycles(slices.Collect(maps.Values(out.Entries)))
slices.Sort(out.Uncomputed)
return out
}
func sortEdges(edges []meshwants.Edge) {
slices.SortFunc(edges, func(a, b meshwants.Edge) int {
return strings.Compare(edgeKey(a), edgeKey(b))
})
}
func edgeKey(e meshwants.Edge) string {
return strings.Join([]string{e.Kind, e.Node, e.Module, e.Provision, e.Seat, e.Step, e.Fp, e.Bound}, "\x00")
}
// EntryDigest is the digest an entry is stored under: the same entry, the same digest.
func EntryDigest(e meshwants.AssignmentEntry) string {
raw, _ := json.Marshal(e)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Same says two revisions list the same entries computed from the same inputs, so the second is no new revision.
func Same(a, b meshwants.AssignmentRevision) bool {
return slices.Equal(a.Entries, b.Entries) && slices.Equal(a.Catalogs, b.Catalogs) && a.Placements == b.Placements &&
slices.EqualFunc(a.Capabilities, b.Capabilities, func(x, y meshwants.AssignmentRevisionCapabilitiesItem) bool {
// A reading's digest is over all of it: its readings and its runner's fingerprint among it.
return x.Node == y.Node && x.Digest == y.Digest
})
}
// resolve is the provider requirement r of p points to by §708's order, or the hold that says why none is: none and no
// hold for an optional requirement nothing provides (ADR 0330 §3).
func resolve(p *placedAt, r string, placed []*placedAt, holders map[string]mma.SeatHolders, pins map[string]mma.Provider) (mma.Provider, *meshwants.Held) {
// 1. The named seat's holder, from the placements' holds as the state seat-holders keeps them; a delivering seat no
// placement holds holds the entry, and no other step is tried.
if def, ok := p.delivering[r]; ok {
node := ""
if def.Scope == seatdef.ScopeNode {
node = p.node
}
key := def.Seat
if node != "" {
key += "." + node
}
if sh := holders[key]; len(sh.Holders) > 0 {
return mma.Provider{Node: sh.Holders[0].Node, Module: sh.Holders[0].Module}, nil
}
return mma.Provider{}, &meshwants.Held{Requirement: r, Seat: def.Seat, Candidates: []meshwants.HeldCandidate{}}
}
// 2. A pin: the placed module's bindings, else its node's pins.
if prov, ok := p.placed.Bindings[r]; ok {
return prov, nil
}
if prov, ok := pins[r]; ok {
return prov, nil
}
// 3. The provider on the node; 4. the only provider.
var here, all []*placedAt
for _, q := range placed {
if q != p && slices.Contains(q.provides, r) {
all = append(all, q)
if q.node == p.node {
here = append(here, q)
}
}
}
switch {
case len(here) == 1:
return mma.Provider{Node: here[0].node, Module: here[0].placed.Module}, nil
case len(here) == 0 && len(all) == 1:
return mma.Provider{Node: all[0].node, Module: all[0].placed.Module}, nil
case len(all) == 0 && p.optional(r):
return mma.Provider{}, nil
}
h := &meshwants.Held{Requirement: r, Candidates: []meshwants.HeldCandidate{}}
for _, q := range all {
h.Candidates = append(h.Candidates, meshwants.HeldCandidate{Node: q.node, Module: q.placed.Module})
}
slices.SortFunc(h.Candidates, func(a, b meshwants.HeldCandidate) int {
return strings.Compare(a.Node+"/"+a.Module, b.Node+"/"+b.Module)
})
return mma.Provider{}, h
}
@@ -1,248 +0,0 @@
package assigngraph
import (
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Cycles are the cycles among the acting waits (design 47 G §272, ADR 0656, A-323): an entry waits for its acting parts in the step
// an edge names, so acting parts that wait for each other, in one step or across steps, wait for ever unless a rule breaks the
// cycle. A cycle within one step is `<step>: a -> b -> a`; one across steps names each edge's step, `a -[compose]-> b -[place]-> a`;
// each by node and module, said once from its smallest member, sorted. A holder's own inputs are no edge here (the graph gives an
// acting edge only from a part to the entries it acts on, never to what the part is built from).
func Cycles(entries []meshwants.AssignmentEntry) []string {
type key struct{ step, at string }
next := map[key][]string{}
steps := map[string]bool{}
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind != EdgeActing {
continue
}
k := key{edge.Step, e.Node + "/" + e.Module}
next[k] = append(next[k], edge.Node+"/"+edge.Module)
steps[edge.Step] = true
}
}
var out []string
for _, step := range slices.Sorted(maps.Keys(steps)) {
const (
open = iota + 1
done
)
state := map[string]int{}
var path []string
var walk func(at string)
walk = func(at string) {
state[at] = open
path = append(path, at)
for _, to := range next[key{step, at}] {
switch state[to] {
case 0:
walk(to)
case open:
from := slices.Index(path, to)
cycle := slices.Clone(path[from:])
// start at the smallest member so one cycle is said once, whichever member is met first
lowest := slices.Index(cycle, slices.Min(cycle))
cycle = append(cycle[lowest:], cycle[:lowest]...)
out = append(out, fmt.Sprintf("%s: %s -> %s", step, strings.Join(cycle, " -> "), cycle[0]))
}
}
path = path[:len(path)-1]
state[at] = done
}
starts := map[string]bool{}
for k := range next {
if k.step == step {
starts[k.at] = true
}
}
for _, at := range slices.Sorted(maps.Keys(starts)) {
if state[at] == 0 {
walk(at)
}
}
}
// Across steps (A-323): the shortest cycle through each entry over every acting edge, said when its edges name two steps or more.
for _, hops := range cyclesThrough(entries, func(_ meshwants.AssignmentEntry, e meshwants.Edge) bool { return e.Kind == EdgeActing }) {
if crossesSteps(hops) {
out = append(out, hopWords(rotated(hops)))
}
}
slices.Sort(out)
return slices.Compact(out)
}
// HeldCycles is, by `<node>/<module>`, the cycle each entry lies on among the edges holds says hold it (A-323, A-330): a want held
// on such a cycle is held for ever, since no rule breaks it. Each cycle starts at the entry and names each edge's step.
func HeldCycles(entries []meshwants.AssignmentEntry, holds func(e meshwants.AssignmentEntry, edge meshwants.Edge) bool) map[string]string {
out := map[string]string{}
for at, hops := range cyclesThrough(entries, holds) {
out[at] = hopWords(hops)
}
return out
}
// Engines is, by node, the module that holds the seat node-engine there, read from the acting edges' seat (A-330).
func Engines(entries []meshwants.AssignmentEntry) map[string]string {
out := map[string]string{}
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind == EdgeActing && edge.Seat == nodeengine.Seat {
out[edge.Node] = edge.Module
}
}
}
return out
}
// Runner is the module that holds the seat mesh-pipeline-runner, read from the acting edges' seat as Engines reads the node-engine
// (A-334); "" when none is read.
func Runner(entries []meshwants.AssignmentEntry) string {
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind == EdgeActing && edge.Seat == meshpipelinerunner.Seat {
return edge.Module
}
}
}
return ""
}
// hop is one edge of a cycle: from the entry at, in step, to the next hop's entry.
type hop struct{ at, step string }
// cyclesThrough is, by `<node>/<module>`, the shortest cycle through each entry over the edges keep takes, found within the
// entry's strongly connected part, each starting at the entry.
func cyclesThrough(entries []meshwants.AssignmentEntry, keep func(meshwants.AssignmentEntry, meshwants.Edge) bool) map[string][]hop {
type out struct{ to, step string }
next := map[string][]out{}
for _, e := range entries {
at := e.Node + "/" + e.Module
for _, edge := range e.Edges {
if keep(e, edge) {
next[at] = append(next[at], out{edge.Node + "/" + edge.Module, edge.Step})
}
}
}
for at := range next {
slices.SortFunc(next[at], func(a, b out) int { return strings.Compare(a.to+"\x00"+a.step, b.to+"\x00"+b.step) })
}
part := strongParts(slices.Sorted(maps.Keys(next)), func(at string) []string {
var to []string
for _, o := range next[at] {
to = append(to, o.to)
}
return to
})
found := map[string][]hop{}
for _, start := range slices.Sorted(maps.Keys(next)) {
// breadth first from start back to it, within its part
prev := map[string]hop{}
queue := []string{start}
seen := map[string]bool{}
for len(queue) > 0 && found[start] == nil {
at := queue[0]
queue = queue[1:]
for _, o := range next[at] {
if part[o.to] != part[start] {
continue
}
if o.to == start {
hops := []hop{{at, o.step}}
for cur := at; cur != start; {
h := prev[cur]
hops = append(hops, h)
cur = h.at
}
slices.Reverse(hops)
found[start] = hops
break
}
if !seen[o.to] {
seen[o.to] = true
prev[o.to] = hop{at, o.step}
queue = append(queue, o.to)
}
}
}
}
return found
}
// strongParts numbers each vertex by its strongly connected part (Tarjan).
func strongParts(vertices []string, succ func(string) []string) map[string]int {
index, low, part := map[string]int{}, map[string]int{}, map[string]int{}
on := map[string]bool{}
var stack []string
n, parts := 0, 0
var visit func(v string)
visit = func(v string) {
index[v], low[v] = n, n
n++
stack = append(stack, v)
on[v] = true
for _, w := range succ(v) {
if _, ok := index[w]; !ok {
visit(w)
low[v] = min(low[v], low[w])
} else if on[w] {
low[v] = min(low[v], index[w])
}
}
if low[v] == index[v] {
parts++
for {
w := stack[len(stack)-1]
stack = stack[:len(stack)-1]
on[w] = false
part[w] = parts
if w == v {
break
}
}
}
}
for _, v := range vertices {
if _, ok := index[v]; !ok {
visit(v)
}
}
return part
}
func crossesSteps(hops []hop) bool {
for _, h := range hops[1:] {
if h.step != hops[0].step {
return true
}
}
return false
}
// rotated is the cycle started at its smallest member, so one cycle is said once.
func rotated(hops []hop) []hop {
lowest := 0
for i, h := range hops {
if h.at < hops[lowest].at {
lowest = i
}
}
return append(slices.Clone(hops[lowest:]), hops[:lowest]...)
}
// hopWords is a cycle with each edge's step: `a -[compose]-> b -[place]-> a`.
func hopWords(hops []hop) string {
var b strings.Builder
for _, h := range hops {
fmt.Fprintf(&b, "%s -[%s]-> ", h.at, h.step)
}
return b.String() + hops[0].at
}
@@ -1,60 +0,0 @@
package assigngraph
import (
"slices"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §272 (AS10), A-323: a cycle among the waits is found within one step and across steps, each edge's step named across
// steps; a chain and provisioning edges are none.
func TestACycleAmongTheWaitsWithinOrAcrossStepsIsFound(t *testing.T) {
act := func(node, module, step string) meshwants.Edge {
return meshwants.Edge{Kind: EdgeActing, Node: node, Module: module, Step: step}
}
entry := func(module string, edges ...meshwants.Edge) meshwants.AssignmentEntry {
return meshwants.AssignmentEntry{Node: "ace", Module: module, Edges: edges}
}
for _, c := range []struct {
name string
entries []meshwants.AssignmentEntry
want []string
}{
{"two parts on each other in one step", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "a", "compose"))},
[]string{"compose: ace/a -> ace/b -> ace/a"}},
{"three in a ring, said once", []meshwants.AssignmentEntry{entry("c", act("ace", "a", "build")), entry("a", act("ace", "b", "build")), entry("b", act("ace", "c", "build"))},
[]string{"build: ace/a -> ace/b -> ace/c -> ace/a"}},
{"the indexer taking in the checker, the checker checking the indexer", []meshwants.AssignmentEntry{entry("checker", act("ace", "indexer", "take-in")), entry("indexer", act("ace", "checker", "check"))},
[]string{"ace/checker -[take-in]-> ace/indexer -[check]-> ace/checker"}},
{"the assigner and the node-engine, across compose and place", []meshwants.AssignmentEntry{entry("assigner", act("ace", "engine", "compose")),
entry("engine", act("ace", "assigner", "place")), entry("app", act("ace", "engine", "compose"), act("ace", "assigner", "place"))},
[]string{"ace/assigner -[compose]-> ace/engine -[place]-> ace/assigner"}},
{"a chain", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "c", "compose")), entry("c")}, nil},
{"provisioning edges are no wait of a step", []meshwants.AssignmentEntry{entry("a", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "b"}), entry("b", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "a"})}, nil},
} {
if got := Cycles(c.entries); !slices.Equal(got, c.want) {
t.Errorf("%s: cycles %v, want %v", c.name, got, c.want)
}
}
}
// A-323, A-330: the node-engine on a node is read from the acting edges' seat, and the cycles that hold a want are those of the
// edges said to hold it, each from the entry, naming each edge's step.
func TestTheHeldCyclesAreThoseOfTheHoldingEdges(t *testing.T) {
entries := []meshwants.AssignmentEntry{
{Node: "ace", Module: "assigner", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "engine", Step: "compose", Seat: "node-engine"}}},
{Node: "ace", Module: "engine", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "assigner", Step: "place", Seat: "mesh-wants"}}},
}
if got := Engines(entries); len(got) != 1 || got["ace"] != "engine" {
t.Errorf("the node-engines %v, want engine on ace", got)
}
all := func(meshwants.AssignmentEntry, meshwants.Edge) bool { return true }
if got := HeldCycles(entries, all); len(got) != 2 || got["ace/engine"] != "ace/engine -[place]-> ace/assigner -[compose]-> ace/engine" {
t.Errorf("held cycles %v", got)
}
// the node-engine's want held by no acting edge: no cycle holds anything
if got := HeldCycles(entries, func(e meshwants.AssignmentEntry, _ meshwants.Edge) bool { return e.Module != "engine" }); len(got) != 0 {
t.Errorf("held cycles with the node-engine's edges holding nothing: %v", got)
}
}
@@ -1,400 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
d, ok := m[name]
if !ok {
return nil, ErrNoObject
}
return d, nil
}
func (m memObjects) Names(context.Context) ([]string, error) {
out := []string{}
for n := range m {
out = append(out, n)
}
return out, nil
}
func half(branch string, revision int64, entries ...meshcatalog.CatalogEntry) meshcatalogclient.Half {
h := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Kind: "trunk", Branch: branch, Revision: revision, Source: revision + 10},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, e := range entries {
h.Entries[e.Module] = e
h.Revision.Entries = append(h.Revision.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: "sha256:" + e.Module + e.Fp})
}
return h
}
// Design 47 *Two graphs, one source of truth* (lines 1353-1366, 1416-1418) and *An acting part's new version acts first*
// (lines 1835-1836): each entry one module version on a node from the catalog graph of the branch its placement names,
// its edges provisioning and acting, a revision computed again from its inputs equal to the one kept.
func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) {
ctx := context.Background()
in := Inputs{
Halves: map[string]meshcatalogclient.Half{"main": half("main", 7,
meshcatalog.CatalogEntry{Module: "node-engine", Fp: "src2:e1", Acts: []meshcatalog.ActingRule{{Seat: "node-engine", Step: "compose", On: "every", Bound: "30m"}}},
meshcatalog.CatalogEntry{Module: "build-go", Fp: "src2:b1", Acts: []meshcatalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
meshcatalog.CatalogEntry{Module: "postgres", Fp: "src2:p1", Provides: []string{"database"}},
meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"database", "secrets"}, Toolchains: map[string]meshcatalog.BuildEdge{"go": {Version: "1"}}},
)},
Placements: map[string]mma.Assignments{
"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "postgres"}, {Module: "build-go"},
{Module: "web", Bindings: map[string]mma.Provider{"database": {Node: "anchor", Module: "postgres"}}}}},
"laptop": {Node: "laptop", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "web", Build: "src2:w0"},
{Module: "photos", Branch: "beta"}, {Module: "ghost"}}},
},
FirstBranch: func(module string) string {
if module == "ghost" {
return ""
}
return "main"
},
}
c := Compute(in)
byNode := map[string]meshwants.AssignmentEntry{}
for _, ref := range c.Revision.Entries {
byNode[ref.Node+"/"+ref.Module] = c.Entries[ref.Digest]
}
webA, webL := byNode["anchor/web"], byNode["laptop/web"]
if len(c.Revision.Entries) != 6 || len(c.Revision.Catalogs) != 1 || c.Revision.Catalogs[0] != (meshwants.CatalogOf{Branch: "main", Revision: 7, Source: 17}) {
t.Fatalf("revision %+v", c.Revision)
}
if webA.Branch != "main" || webA.Fp != "src2:w1" || webA.Catalog != "sha256:websrc2:w1" || webL.Fp != "src2:w0" {
t.Errorf("web's entries: %+v, %+v (the placement's build is the version on laptop)", webA, webL)
}
want := []meshwants.Edge{
{Kind: EdgeActing, Node: "anchor", Module: "build-go", Fp: "src2:b1", Seat: "node-build-go", Step: "build"},
{Kind: EdgeActing, Node: "anchor", Module: "node-engine", Fp: "src2:e1", Seat: "node-engine", Step: "compose", Bound: "30m"},
{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"},
}
if fmt.Sprint(webA.Edges) != fmt.Sprint(want) {
t.Errorf("web on anchor's edges:\n got %+v\nwant %+v", webA.Edges, want)
}
// laptop's web: its own node's node-engine composes it, the build module placed once builds it, and with no binding or
// pin its database is the only provider's (design 47 G §357).
if e := webL.Edges; len(e) != 3 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" ||
e[2] != (meshwants.Edge{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"}) {
t.Errorf("web on laptop's edges: %+v", e)
}
// Its secrets no placed module provides: held on both nodes, naming no candidate.
if h := webA.Held; len(h) != 1 || h[0].Requirement != "secrets" || len(h[0].Candidates) != 0 {
t.Errorf("web on anchor is held on %+v", h)
}
if e := byNode["anchor/postgres"].Edges; len(e) != 1 || e[0].Module != "node-engine" {
t.Errorf("postgres builds no Go, so only the node-engine acts on it: %+v", e)
}
if len(byNode["anchor/node-engine"].Edges) != 0 {
t.Errorf("an acting part acts on itself: %+v", byNode["anchor/node-engine"].Edges)
}
if strings.Join(c.Uncomputed, "|") != "laptop/ghost: no watched branch: its placement names none and no registration of it is known|"+
"laptop/photos: the catalog graph of the branch beta is not held" {
t.Errorf("uncomputed %q", c.Uncomputed)
}
// Kept once; computed again from the same inputs it is the same revision, kept no second time.
o := memObjects{}
at := time.Date(2026, 11, 10, 18, 0, 0, 0, time.UTC)
rev, kept, err := Keep(ctx, o, c, "catalog-revision", at)
if err != nil || !kept || rev.Revision != 1 || rev.Cause != "catalog-revision" || len(o) != 7 {
t.Fatalf("kept %v %+v (%v), objects %d", kept, rev, err, len(o))
}
if again, kept, err := Keep(ctx, o, Compute(in), "placement-changed", at); err != nil || kept || again.Revision != 1 {
t.Errorf("recomputed from the same inputs: kept %v, %+v, %v", kept, again, err)
}
// A placement change moves laptop's web to the catalog's version: revision 2, one entry more stored, the rest shared.
in.Placements["laptop"].Modules[1].Build = ""
rev2, kept, err := Keep(ctx, o, Compute(in), "placement-changed", at)
if err != nil || !kept || rev2.Revision != 2 || len(o) != 9 || rev2.Placements == rev.Placements {
t.Fatalf("revision 2 %v %+v (%v), objects %d", kept, rev2, err, len(o))
}
// The verbs read only what is kept.
r, err := Revision(ctx, o, meshwants.RevisionInput{})
if err != nil || r.Revision.Revision != 2 {
t.Errorf("revision = %+v, %v", r, err)
}
if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "g", Heads: "h"}); !errors.Is(err, ErrNoCandidate) {
t.Errorf("a candidate's revision: %v", err)
}
e, err := Entry(ctx, o, meshwants.EntryInput{Revision: 1, Node: "laptop", Module: "web"})
if err != nil || e.Entry.Fp != "src2:w0" || !strings.HasPrefix(e.Digest, "sha256:") {
t.Errorf("entry = %+v, %v", e, err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{From: 1, To: 2})
if err != nil || len(d.Changes) != 1 || fmt.Sprint(d.Changes[0]) != fmt.Sprint(meshwants.Change{Node: "laptop", Module: "web", From: "src2:w0", To: "src2:w1"}) ||
strings.Join(d.Nodes, ",") != "laptop" {
t.Errorf("difference = %+v, %v", d, err)
}
if _, err := Difference(ctx, o, meshwants.DifferenceInput{To: 2}); err == nil {
t.Error("a difference with one revision answered")
}
if ev := Event(rev2); ev.Revision != 2 || len(ev.Catalogs) != 1 || ev.Kind != "trunk" {
t.Errorf("event %+v", ev)
}
}
// Design 47 G §357, §708, §709 and A-167, A-229: a requirement points to the named seat's holder, else a pin, else the
// provider on the consumer's node, else the only provider; otherwise the entry is held, naming the candidates (and the
// unheld seat), and nothing is guessed. An optional requirement nothing provides is left absent; a requirement the
// manifest's contributes names gets a contribution edge to the same provider.
func TestARequirementPointsByTheOrderOrIsHeld(t *testing.T) {
for _, c := range []struct {
name string
kv []string // the nodes kv is placed on
pin bool // web's node pins kv on laptop
seat string // the seat delivering cache: "" none, "held" held by kv on server, "unheld" held by none
optional bool // web requires cache optionally
contributes bool // web's manifest contributes to cache
edge string // the node of the provisioning edge, "" for none
held string // the candidates' nodes of the hold, "-" for no hold
heldSeat string // the seat the hold names
}{
{name: "the provider on the consumer's node", kv: []string{"anchor", "laptop"}, edge: "anchor", held: "-"},
{name: "the only provider", kv: []string{"laptop"}, edge: "laptop", held: "-"},
{name: "a pin before the node's own", kv: []string{"anchor", "laptop"}, pin: true, edge: "laptop", held: "-"},
{name: "two providers elsewhere: held", kv: []string{"laptop", "server"}, held: "laptop,server"},
{name: "none: held naming none", held: ""},
{name: "the named seat's holder before the provider on the node", kv: []string{"anchor", "server"}, seat: "held", edge: "server", held: "-"},
{name: "a delivering seat no placement holds: held naming it", kv: []string{"anchor"}, seat: "unheld", held: "", heldSeat: "cache-seat"},
{name: "an optional requirement nothing provides: absent", optional: true, held: "-"},
{name: "an optional requirement with two providers elsewhere: held", optional: true, kv: []string{"laptop", "server"}, held: "laptop,server"},
{name: "a contributed requirement: a contribution edge too", kv: []string{"laptop"}, contributes: true, edge: "laptop", held: "-"},
} {
t.Run(c.name, func(t *testing.T) {
web := meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"cache"},
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"cache"}}}
if c.optional {
web.Manifest["requires"] = []any{map[string]any{"name": "cache", "optional": true}}
}
if c.contributes {
web.Contributes = []string{"cache"}
}
h := half("main", 3, meshcatalog.CatalogEntry{Module: "kv", Fp: "src2:k1", Provides: []string{"cache"}}, web)
defs := map[string]seatdef.Definition{}
if c.seat != "" {
h.Revision.Definitions = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "cache-seat", Digest: "sha256:d"}}
defs["cache-seat"] = seatdef.Definition{Seat: "cache-seat", Scope: seatdef.ScopeMesh, Delivers: "cache"}
}
pl := map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}}
if c.pin {
pl["anchor"] = mma.Assignments{Node: "anchor", Modules: pl["anchor"].Modules, Pins: map[string]mma.Provider{"cache": {Node: "laptop", Module: "kv"}}}
}
for _, n := range c.kv {
a := pl[n]
a.Node, a.Modules = n, append(a.Modules, mma.PlacedModule{Module: "kv"})
pl[n] = a
}
if c.seat == "held" {
a := pl["server"]
a.Holds = append(a.Holds, mma.Holding{Claim: "cache-seat", Module: "kv"})
pl["server"] = a
}
out := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: pl, FirstBranch: func(string) string { return "main" },
Definitions: defs})
var got meshwants.AssignmentEntry
for _, ref := range out.Revision.Entries {
if ref.Node == "anchor" && ref.Module == "web" {
got = out.Entries[ref.Digest]
}
}
edge, contribution := "", ""
for _, e := range got.Edges {
if e.Kind == EdgeProvisioning && e.Provision == "cache" {
edge = e.Node
}
if e.Kind == EdgeContribution && e.Provision == "cache" {
contribution = e.Node + "/" + e.Module
}
}
held, heldSeat := "-", ""
if len(got.Held) == 1 && got.Held[0].Requirement == "cache" {
var nodes []string
for _, cand := range got.Held[0].Candidates {
nodes = append(nodes, cand.Node)
}
held, heldSeat = strings.Join(nodes, ","), got.Held[0].Seat
}
if edge != c.edge || held != c.held || heldSeat != c.heldSeat || len(got.Held) > 1 {
t.Errorf("web points to %q, held on %q seat %q (%+v); want %q, %q, %q", edge, held, heldSeat, got.Held, c.edge, c.held, c.heldSeat)
}
if want := map[bool]string{true: c.edge + "/kv"}[c.contributes]; contribution != want {
t.Errorf("web's contribution edge points to %q, want %q", contribution, want)
}
})
}
}
// Design 47 line 1385: a whole assignment graph revision of about 120 modules on four nodes is under the bus's 256 KiB
// value limit.
func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) {
var entries []meshcatalog.CatalogEntry
placements := map[string]mma.Assignments{}
for i := range 120 {
entries = append(entries, meshcatalog.CatalogEntry{Module: fmt.Sprintf("module-%03d", i), Fp: "src2:" + strings.Repeat("a", 64)})
}
for _, node := range []string{"anchor", "home", "desk", "laptop"} {
a := mma.Assignments{Node: node}
for _, e := range entries {
a.Modules = append(a.Modules, mma.PlacedModule{Module: e.Module})
}
placements[node] = a
}
h := half("main", 1, entries...)
for i := range h.Revision.Entries {
h.Revision.Entries[i].Digest = "sha256:" + strings.Repeat("b", 64)
}
c := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: placements, FirstBranch: func(string) string { return "main" }})
raw, _ := json.Marshal(c.Revision)
if len(c.Revision.Entries) != 480 || len(raw) >= 256<<10 {
t.Errorf("%d entries in %d bytes", len(c.Revision.Entries), len(raw))
}
}
func (m memObjects) Delete(_ context.Context, name string) error {
delete(m, name)
return nil
}
// A-201, A-235, §186, §189: a run's candidate assignment graph is kept apart from trunk's under candidate.<group>.<heads>, served by
// revision, entry and difference against trunk, and dropped by its group alone.
func TestARunsCandidateGraphIsKeptApartServedAndDropped(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
placed := map[string]mma.Assignments{"ace": {Modules: []mma.PlacedModule{{Module: "web"}}}}
first := func(string) string { return "main" }
o := memObjects{}
trunk := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 3, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1"})},
Placements: placed, FirstBranch: first})
if _, _, err := Keep(ctx, o, trunk, "catalog-revision", at); err != nil {
t.Fatal(err)
}
cand := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 4, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w2"})},
Placements: placed, FirstBranch: first})
if _, _, err := KeepCandidate(ctx, o, cand, "", "h1", "catalog-revision", at); err == nil {
t.Error("a candidate naming no group was kept")
}
rev, fresh, err := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at)
if err != nil || !fresh || rev.Kind != meshcatalogclient.KindCandidate || rev.Group != "run-7" || rev.Revision != 1 {
t.Fatalf("kept %+v fresh %v: %v", rev, fresh, err)
}
if _, ok := o[CandidateObject("run-7", "h1")]; !ok {
t.Fatalf("the candidate is not under candidate.run-7.h1: %v", o)
}
if _, fresh, _ := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at); fresh {
t.Error("the same candidate kept again is a new revision")
}
newest, has, err := Newest(ctx, o)
if err != nil || !has || newest.Kind != meshcatalogclient.KindTrunk || newest.Revision != 1 {
t.Errorf("trunk's newest is %+v (%v, %v): the candidate is not kept apart", newest, has, err)
}
for _, in := range []meshwants.RevisionInput{{Group: "run-7"}, {Group: "run-7", Heads: "h1"}} {
a, err := Revision(ctx, o, in)
if err != nil || a.Revision.Group != "run-7" {
t.Errorf("revision %+v answered %+v: %v", in, a.Revision, err)
}
}
if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "run-8"}); !errors.Is(err, ErrNoCandidate) {
t.Errorf("an unkept candidate answered %v", err)
}
e, err := Entry(ctx, o, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"})
if err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the candidate's entry is %+v: %v", e.Entry, err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{Group: "run-7"})
if err != nil || len(d.Changes) != 1 || d.Changes[0].From != "src2:w1" || d.Changes[0].To != "src2:w2" {
t.Errorf("the candidate against trunk is %+v: %v", d, err)
}
if _, err := Difference(ctx, o, meshwants.DifferenceInput{}); err == nil {
t.Error("difference with none of from, to and group is not refused")
}
if _, _, err := KeepCandidate(ctx, o, cand, "run-9", "h2", "catalog-revision", at); err != nil {
t.Fatal(err)
}
if n, err := DropCandidate(ctx, o, "run-7"); err != nil || n != 1 {
t.Errorf("dropped %d: %v", n, err)
}
if _, err := ReadCandidate(ctx, o, "run-7", ""); !errors.Is(err, ErrNoCandidate) {
t.Errorf("the dropped candidate reads: %v", err)
}
if _, err := ReadCandidate(ctx, o, "run-9", ""); err != nil {
t.Errorf("another run's candidate went with it: %v", err)
}
if _, has, _ := Newest(ctx, o); !has {
t.Error("dropping a candidate dropped trunk")
}
}
// Design 47 G §512, §186, A-242: the difference orders its changes itself: each change names the changes it is ordered after
// with the kind of the edge, and Tiers holds the changes in that order, an acting part and a provider before what they act on.
func TestTheDifferenceFillsItsTiersAndEachChangesAfter(t *testing.T) {
ctx := context.Background()
build := func(ne, pg, web string) Computed {
return Compute(Inputs{
Halves: map[string]meshcatalogclient.Half{"main": half("main", 7,
meshcatalog.CatalogEntry{Module: "node-engine", Fp: ne, Acts: []meshcatalog.ActingRule{{Seat: "node-engine", Step: "compose", On: "every"}}},
meshcatalog.CatalogEntry{Module: "postgres", Fp: pg, Provides: []string{"database"}},
meshcatalog.CatalogEntry{Module: "web", Fp: web, Requires: []string{"database"}},
)},
Placements: map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "postgres"},
{Module: "web", Bindings: map[string]mma.Provider{"database": {Node: "anchor", Module: "postgres"}}}}}},
FirstBranch: func(string) string { return "main" },
})
}
o := memObjects{}
at := time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)
if _, _, err := Keep(ctx, o, build("src2:e1", "src2:p1", "src2:w1"), "catalog-revision", at); err != nil {
t.Fatal(err)
}
if _, _, err := Keep(ctx, o, build("src2:e2", "src2:p2", "src2:w2"), "catalog-revision", at); err != nil {
t.Fatal(err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{From: 1, To: 2})
if err != nil || len(d.Changes) != 3 {
t.Fatalf("difference %+v, %v", d, err)
}
after := map[string]string{}
for _, c := range d.Changes {
var a []string
for _, x := range c.After {
a = append(a, x.Kind+":"+x.Node+"/"+x.Module)
}
after[c.Module] = strings.Join(a, ",")
}
if after["node-engine"] != "" || after["postgres"] != "acting:anchor/node-engine" ||
after["web"] != "acting:anchor/node-engine,provisioning:anchor/postgres" {
t.Errorf("after = %v", after)
}
var tiers []string
for _, tier := range d.Tiers {
var ms []string
for _, r := range tier {
ms = append(ms, r.Module)
}
tiers = append(tiers, strings.Join(ms, ","))
}
if strings.Join(tiers, "|") != "node-engine|postgres|web" {
t.Errorf("tiers = %v", tiers)
}
}
@@ -1,185 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// RevisionsObjects is the local name of the kept data in the assigner's manifest's old `objects` section (design 47
// lines 1362-1366: both foundation parts keep their revisions as objects in the old form; the seat mesh-wants' state
// assignment-revisions names its key forms).
const RevisionsObjects = "assignment-revisions"
// ErrNoObject is an object the bucket does not hold.
var ErrNoObject = errors.New("no such object")
// Objects is an object bucket as the assigner keeps its revisions in it.
type Objects interface {
// Put stores data whole under name, replacing one of that name.
Put(ctx context.Context, name string, data []byte) error
// Get is the object's bytes; ErrNoObject when it holds none of that name.
Get(ctx context.Context, name string) ([]byte, error)
// Names is every object's name; none is an empty list.
Names(ctx context.Context) ([]string, error)
}
// EntryObject is the object name of the entry of that digest (the seat's key form entry.<digest>).
func EntryObject(digest string) string { return "entry." + digest }
// TrunkObject is the object name of a trunk revision (the seat's key form trunk.<revision>).
func TrunkObject(revision int64) string { return "trunk." + strconv.FormatInt(revision, 10) }
// trunkOf is the revision number a trunk object's name holds, and whether it is one.
func trunkOf(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// Newest is the newest trunk revision kept, its list only; none kept is false.
func Newest(ctx context.Context, o Objects) (meshwants.AssignmentRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var newest int64
for _, n := range names {
if r, ok := trunkOf(n); ok && r > newest {
newest = r
}
}
if newest == 0 {
return meshwants.AssignmentRevision{}, false, nil
}
rev, err := ReadRevision(ctx, o, newest)
return rev, err == nil, err
}
// ReadRevision is the trunk revision of that number, its list only.
func ReadRevision(ctx context.Context, o Objects, revision int64) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, TrunkObject(revision))
if err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
return rev, nil
}
// ReadEntry is the entry kept under that digest.
func ReadEntry(ctx context.Context, o Objects, digest string) (meshwants.AssignmentEntry, error) {
var e meshwants.AssignmentEntry
data, err := o.Get(ctx, EntryObject(digest))
if err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
if err := json.Unmarshal(data, &e); err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
return e, nil
}
// Keep keeps c as the next trunk revision, numbered after the newest kept, unless the newest kept lists the same entries
// from the same inputs (Same): then it keeps nothing and answers the newest and false. Each entry object not held yet is
// written before the list, so a list is never kept whose entries are not, and an entry two revisions share is stored
// once (design 47 *An unchanged entry is stored once*).
func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
newest, has, err := Newest(ctx, o)
if err != nil {
return newest, false, err
}
if has && Same(newest, c.Revision) {
return newest, false, nil
}
rev := c.Revision
rev.Revision = newest.Revision + 1
rev.Cause = cause
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return newest, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return newest, false, err
}
if err := o.Put(ctx, TrunkObject(rev.Revision), list); err != nil {
return newest, false, fmt.Errorf("revision %d in %s: %w", rev.Revision, RevisionsObjects, err)
}
return rev, true, nil
}
// writeEntries writes each entry object rev lists that the bucket does not hold yet, from c, before its list is kept.
func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.AssignmentRevision) error {
names, err := o.Names(ctx)
if err != nil {
return fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if held[name] {
continue
}
e, ok := c.Entries[ref.Digest]
if !ok {
return fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest)
}
data, err := json.Marshal(e)
if err != nil {
return err
}
if err := o.Put(ctx, name, data); err != nil {
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
}
held[name] = true
}
return nil
}
// Event is the seat's event assignment-revision of a kept revision.
func Event(rev meshwants.AssignmentRevision) meshwants.AssignmentRevisionEvent {
return meshwants.AssignmentRevisionEvent{V: rev.V, Revision: rev.Revision, Kind: rev.Kind, Cause: rev.Cause, Group: rev.Group,
Heads: rev.Heads, Catalogs: rev.Catalogs, At: rev.At}
}
// Filler is an object bucket that says its fill: the bytes used and its cap in bytes (0: no cap known).
type Filler interface {
Fill(ctx context.Context) (used, capacity int64, err error)
}
// NewBytes is what keeping c next would add to the bucket: each entry object not held yet and the list (design 47 G §181).
func NewBytes(ctx context.Context, o Objects, c Computed) int64 {
names, err := o.Names(ctx)
if err != nil {
return 0
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
var size int64
for _, ref := range c.Revision.Entries {
if e, ok := c.Entries[ref.Digest]; ok && !held[EntryObject(ref.Digest)] {
data, _ := json.Marshal(e)
size += int64(len(data))
}
}
list, _ := json.Marshal(c.Revision)
return size + int64(len(list))
}
@@ -1,6 +0,0 @@
package assigngraph
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
@@ -1,81 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"fmt"
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Reverted says a want of module on node at wantFp is a revert (design 47 G §298, §300, A-217): the module had wantFp in an earlier
// assignment graph revision than the one it now runs from, by the revisions' own numbers. The revision it runs from is the first kept
// that lists runFp for the module on the node; a want is a revert when it was listed before that, and never listed since.
func Reverted(ctx context.Context, o Objects, node, module, runFp, wantFp string) (bool, error) {
if runFp == "" || wantFp == "" || runFp == wantFp {
return false, nil
}
names, err := o.Names(ctx)
if err != nil {
return false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var numbers []int64
for _, n := range names {
if r, ok := trunkOf(n); ok {
numbers = append(numbers, r)
}
}
slices.Sort(numbers)
var ranFrom, lastWanted int64
for _, r := range numbers {
rev, err := ReadRevision(ctx, o, r)
if err != nil {
return false, err
}
for _, ref := range rev.Entries {
if ref.Node != node || ref.Module != module {
continue
}
e, err := ReadEntry(ctx, o, ref.Digest)
if err != nil {
return false, err
}
switch e.Fp {
case runFp:
if ranFrom == 0 {
ranFrom = r
}
case wantFp:
lastWanted = r
}
}
}
return ranFrom != 0 && lastWanted != 0 && lastWanted < ranFrom, nil
}
// FormChanges are the kept data whose `form` differs between the running entry's manifest and the want's (design 47 G §295 to §298,
// A-217): the entry's manifest whole is read as a manifest and compared by catalogue.FormChanges, so a kept data's form is read from
// the entry's kept declarations (catalogue.KeptDeclarations), never from a field of the want.
func FormChanges(running, applied meshcatalog.CatalogEntry) ([]catalogue.FormChange, error) {
before, err := manifestOf(running)
if err != nil {
return nil, fmt.Errorf("the running manifest of %s: %w", running.Module, err)
}
after, err := manifestOf(applied)
if err != nil {
return nil, fmt.Errorf("the manifest of %s: %w", applied.Module, err)
}
return catalogue.FormChanges(before, after), nil
}
func manifestOf(e meshcatalog.CatalogEntry) (catalogue.Manifest, error) {
var m catalogue.Manifest
raw, err := json.Marshal(e.Manifest)
if err != nil {
return m, err
}
err = json.Unmarshal(raw, &m)
return m, err
}
@@ -1,49 +0,0 @@
package assigngraph
import (
"context"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// Design 47 G §298, §300, A-217 (AS17): a want is a revert when the module had its fingerprint in an earlier assignment graph revision than
// the one it now runs from; a kept data whose form changed is read from the entries' manifests.
func TestAWantGoingBackToAnEarlierFingerprintIsARevert(t *testing.T) {
ctx := context.Background()
o := memObjects{}
at := time.Now()
for _, fp := range []string{"src2:a", "src2:b", "src2:c"} {
in := Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 1, meshcatalog.CatalogEntry{Module: "web", Fp: fp})},
Placements: map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}}}, FirstBranch: func(string) string { return "main" }}
if _, _, err := Keep(ctx, o, Compute(in), "test", at); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
run, want string
revert bool
}{{"src2:c", "src2:a", true}, {"src2:c", "src2:b", true}, {"src2:a", "src2:c", false}, {"src2:b", "src2:b", false}, {"src2:c", "src2:zz", false}} {
if got, err := Reverted(ctx, o, "ace", "web", c.run, c.want); err != nil || got != c.revert {
t.Errorf("runs %s, wanted %s: revert %v %v, want %v", c.run, c.want, got, err, c.revert)
}
}
}
func TestAKeptDatasFormIsReadFromTheEntrysManifest(t *testing.T) {
entry := func(form int) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "web", Manifest: map[string]any{"module": "web",
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
}
same, err := FormChanges(entry(1), entry(1))
if err != nil || len(same) != 0 {
t.Fatalf("same form: %v %v", same, err)
}
got, err := FormChanges(entry(1), entry(2))
if err != nil || len(got) != 1 || got[0].Name != "sessions" {
t.Errorf("changed form: %+v %v", got, err)
}
}
@@ -1,258 +0,0 @@
package assigngraph
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// ErrNoCandidate answers a verb asked of a candidate's revision the assigner keeps none of: a run's candidate is kept from its
// catalog graph revision until its run's group-ended (A-201, A-235).
var ErrNoCandidate = errors.New("the assigner keeps no such candidate's assignment graph revision")
// ErrNoRevision answers a verb asked while no trunk revision is kept yet.
var ErrNoRevision = errors.New("no assignment graph revision is kept yet")
func revisionOf(ctx context.Context, o Objects, revision int64, group, heads string) (meshwants.AssignmentRevision, error) {
if group != "" || heads != "" {
if group == "" {
return meshwants.AssignmentRevision{}, fmt.Errorf("a candidate's revision is asked by its group, with or without its heads, not by heads %q alone", heads)
}
rev, err := ReadCandidate(ctx, o, group, heads)
if err == nil && revision > 0 && rev.Revision != revision {
err = fmt.Errorf("%w (group %q, revision %d: the one kept is %d)", ErrNoCandidate, group, revision, rev.Revision)
}
return rev, err
}
if revision > 0 {
return ReadRevision(ctx, o, revision)
}
rev, has, err := Newest(ctx, o)
if err == nil && !has {
err = ErrNoRevision
}
return rev, err
}
// Revision answers the seat's verb revision: one kept revision, the trunk's newest when none is named.
func Revision(ctx context.Context, o Objects, in meshwants.RevisionInput) (meshwants.RevisionAnswer, error) {
rev, err := revisionOf(ctx, o, in.Revision, in.Group, in.Heads)
return meshwants.RevisionAnswer{Revision: rev}, err
}
// Entry answers the seat's verb entry: one entry whole, by its digest, or by its node and module in a revision (the
// trunk's newest when none is named); an acting edge names the seat whose rule gave it.
func Entry(ctx context.Context, o Objects, in meshwants.EntryInput) (meshwants.EntryAnswer, error) {
digest := in.Digest
if digest == "" {
if in.Node == "" || in.Module == "" {
return meshwants.EntryAnswer{}, errors.New("entry takes a digest, or a node and a module")
}
rev, err := revisionOf(ctx, o, in.Revision, in.Group, in.Heads)
if err != nil {
return meshwants.EntryAnswer{}, err
}
for _, ref := range rev.Entries {
if ref.Node == in.Node && ref.Module == in.Module {
digest = ref.Digest
}
}
if digest == "" {
return meshwants.EntryAnswer{}, fmt.Errorf("revision %d has no entry of %s on %s", rev.Revision, in.Module, in.Node)
}
}
e, err := ReadEntry(ctx, o, digest)
return meshwants.EntryAnswer{Digest: digest, Entry: e}, err
}
// Difference answers the seat's verb difference: two kept revisions compared, each module on a node whose version
// differs, with the edges that moved, and the nodes it touches.
func Difference(ctx context.Context, o Objects, in meshwants.DifferenceInput) (meshwants.Difference, error) {
if in.Group != "" || in.Heads != "" {
// A candidate against trunk (§186, §187): from the trunk revision named, else the newest kept.
to, err := revisionOf(ctx, o, 0, in.Group, in.Heads)
if err != nil {
return meshwants.Difference{}, err
}
from, err := revisionOf(ctx, o, in.From, "", "")
if err != nil {
return meshwants.Difference{}, err
}
return Compare(ctx, o, from, to)
}
if in.From <= 0 || in.To <= 0 {
return meshwants.Difference{}, errors.New("difference takes the two kept revisions it compares, from and to")
}
from, err := ReadRevision(ctx, o, in.From)
if err != nil {
return meshwants.Difference{}, err
}
to, err := ReadRevision(ctx, o, in.To)
if err != nil {
return meshwants.Difference{}, err
}
return Compare(ctx, o, from, to)
}
// Compare is the graph difference of two revisions.
func Compare(ctx context.Context, o Objects, from, to meshwants.AssignmentRevision) (meshwants.Difference, error) {
d := meshwants.Difference{From: from.Revision, To: to.Revision, Changes: []meshwants.Change{}}
was, err := entriesOf(ctx, o, from)
if err != nil {
return d, err
}
now, err := entriesOf(ctx, o, to)
if err != nil {
return d, err
}
nodes := map[string]bool{}
edges := map[string][]meshwants.Edge{}
for _, k := range slices.Sorted(maps.Keys(union(was, now))) {
a, hadA := was[k]
b, hasB := now[k]
if hadA && hasB && a.Fp == b.Fp {
continue
}
c := meshwants.Change{}
if hadA {
c.Node, c.Module, c.From = a.Node, a.Module, a.Fp
}
if hasB {
c.Node, c.Module, c.To = b.Node, b.Module, b.Fp
}
c.Edges = moved(a.Edges, b.Edges)
d.Changes = append(d.Changes, c)
nodes[c.Node] = true
if hasB {
edges[c.Node+"/"+c.Module] = b.Edges
} else {
edges[c.Node+"/"+c.Module] = a.Edges
}
}
d.Nodes = slices.Sorted(maps.Keys(nodes))
order(&d, edges, digests(from, to))
return d, nil
}
// digests are the entry digests of the two revisions by node and module, the newer's winning.
func digests(from, to meshwants.AssignmentRevision) map[string]string {
out := map[string]string{}
for _, r := range []meshwants.AssignmentRevision{from, to} {
for _, ref := range r.Entries {
out[ref.Node+"/"+ref.Module] = ref.Digest
}
}
return out
}
// order fills each change's After and the difference's Tiers (design 47 G §512, §186, A-242): a change is ordered after
// each other change of the difference that an edge of its entry names (the module version it is provisioned by, contributes
// to or is acted on by), with that edge's kind; its tier is one past the highest tier it is after. Changes on a cycle come in
// the last tier. The view orders nothing itself.
func order(d *meshwants.Difference, edges map[string][]meshwants.Edge, digest map[string]string) {
if len(d.Changes) == 0 {
return
}
in := map[string]int{}
for i, c := range d.Changes {
in[c.Node+"/"+c.Module] = i
}
for i, c := range d.Changes {
self := c.Node + "/" + c.Module
seen := map[string]bool{}
for _, e := range edges[self] {
k := e.Node + "/" + e.Module
if _, ok := in[k]; !ok || k == self || seen[e.Kind+k] {
continue
}
seen[e.Kind+k] = true
d.Changes[i].After = append(d.Changes[i].After, meshwants.ChangeAfterItem{Node: e.Node, Module: e.Module, Kind: e.Kind})
}
slices.SortFunc(d.Changes[i].After, func(a, b meshwants.ChangeAfterItem) int {
return strings.Compare(a.Kind+"\x00"+a.Node+"\x00"+a.Module, b.Kind+"\x00"+b.Node+"\x00"+b.Module)
})
}
tier := map[int]int{}
var level func(int, map[int]bool) int
level = func(i int, visiting map[int]bool) int {
if t, ok := tier[i]; ok {
return t
}
if visiting[i] {
return -1
}
visiting[i] = true
t := 0
for _, a := range d.Changes[i].After {
l := level(in[a.Node+"/"+a.Module], visiting)
if l < 0 {
return -1
}
t = max(t, l+1)
}
delete(visiting, i)
tier[i] = t
return t
}
top := 0
for i := range d.Changes {
top = max(top, level(i, map[int]bool{}))
}
for i := range d.Changes {
if _, ok := tier[i]; !ok {
tier[i] = top + 1
}
}
for i, c := range d.Changes {
for len(d.Tiers) <= tier[i] {
d.Tiers = append(d.Tiers, nil)
}
d.Tiers[tier[i]] = append(d.Tiers[tier[i]], meshwants.NodeEntryRef{Node: c.Node, Module: c.Module, Digest: digest[c.Node+"/"+c.Module]})
}
d.Tiers = slices.DeleteFunc(d.Tiers, func(t []meshwants.NodeEntryRef) bool { return len(t) == 0 })
}
func entriesOf(ctx context.Context, o Objects, rev meshwants.AssignmentRevision) (map[string]meshwants.AssignmentEntry, error) {
out := make(map[string]meshwants.AssignmentEntry, len(rev.Entries))
for _, ref := range rev.Entries {
e, err := ReadEntry(ctx, o, ref.Digest)
if err != nil {
return nil, fmt.Errorf("revision %d: %w", rev.Revision, err)
}
out[ref.Node+"/"+ref.Module] = e
}
return out, nil
}
func union(a, b map[string]meshwants.AssignmentEntry) map[string]bool {
out := map[string]bool{}
for k := range a {
out[k] = true
}
for k := range b {
out[k] = true
}
return out
}
// moved are the edges in one list and not the other.
func moved(a, b []meshwants.Edge) []meshwants.Edge {
var out []meshwants.Edge
for _, e := range b {
if !slices.Contains(a, e) {
out = append(out, e)
}
}
for _, e := range a {
if !slices.Contains(b, e) {
out = append(out, e)
}
}
sortEdges(out)
return out
}
@@ -1,360 +0,0 @@
package assignment
import (
"encoding/json"
"errors"
"reflect"
"slices"
"strings"
"testing"
"time"
client "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ADR 0567 §2, §4: what a change says, the holder's check of every assignments, a proposal's changes laid over the keys,
// its digests (held to the values the earlier holder's code made, in the field order it wrote), its refusals, and who holds
// which seat.
func TestAssignments(t *testing.T) {
g14 := seat.Assignments{Node: "g14", Revision: "abc",
Decided: seat.NodeDecided{Endpoint: "g14.example:51820", Site: "home", Address: "10.77.0.2", Account: "op", Taken: []string{"zsh"},
Tunnel: &seat.FoundTunnel{Interface: "wg0", Config: "c", Port: 51820, Mtu: 1420, PublicKey: "K", Peers: []seat.FoundTunnelPeer{{PublicKey: "P", Address: "10.0.0.3"}}},
PeerNames: map[string]string{"P": "pi"}},
Modules: []seat.PlacedModule{
{Module: "zsh"},
{Module: "app", Ports: map[string]int64{"8080": 18080}, Bindings: map[string]seat.Provider{"postgres": {Node: "novox", Module: "postgres"}}},
},
Pins: map[string]seat.Provider{"shell": {Node: "g14", Module: "zsh"}},
Holds: []seat.Holding{{Claim: "node-shell", Scope: "node", Module: "zsh"}},
}
novox := seat.Assignments{Node: "novox", Decided: seat.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []seat.PlacedModule{{Module: "postgres", Ports: map[string]int64{"5432": 5432}}},
Holds: []seat.Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}}
current := map[string]seat.Assignments{"g14": g14, "novox": novox}
const id = "p-20261028t183000-1a2b3c"
// the enroller is whichever module holds the seat mesh-node-enroller: no module is named for it
const enroller = "enroll-lab"
enrollers := EnrollersOf(map[string]seat.Assignments{"ace": {Node: "ace",
Holds: []seat.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: enroller}}}})
if !slices.Equal(enrollers, []string{enroller}) {
t.Fatalf("the holders of the enroller seat = %v", enrollers)
}
t.Run("what a change says", func(t *testing.T) {
after := g14
after.Modules = []seat.PlacedModule{{Module: "app", Ports: map[string]int64{"8080": 18081}}, {Module: "new"}}
got := ChangeOf(g14, after)
want := Change{Node: "g14", Added: []string{"new"}, Removed: []string{"zsh"}, Changed: []string{"app"}}
if !reflect.DeepEqual(got, want) {
t.Errorf("ChangeOf = %+v, want %+v", got, want)
}
pinned := g14
pinned.Pins = nil
if got := ChangeOf(g14, pinned); !reflect.DeepEqual(got.Changed, []string{"app", "zsh"}) {
t.Errorf("a pin change changes every module on the node: %+v", got)
}
if got := ChangeOf(seat.Assignments{}, novox); !reflect.DeepEqual(got.Added, []string{"postgres"}) || got.Node != "novox" {
t.Errorf("a new node: %+v", got)
}
c := []Change{{Node: "novox"}, {Node: "g14", Before: 1, After: 2}}
SortChanges(c)
if c[0].Node != "g14" || c[0].Body()["after"] != float64(2) {
t.Errorf("sorted %+v, body %v", c, c[0].Body())
}
})
t.Run("the check of every assignments", func(t *testing.T) {
known := func(m string) bool { return m != "ghost" }
enrolled := func(n string) bool { return n != "stranger" }
if r := CheckAssignments(current, known, enrolled, true); len(r) != 0 {
t.Fatalf("a sound assignments is refused: %v", r)
}
// A replicated mesh seat (ADR 0223) is held on two nodes; a seat that is not replicated is refused there. Issue 1966:
// a seat the list does not name, while the list is unread, waits for it and is never said held once (ADR 0636 §1);
// once read, it is judged held once.
for _, c := range []struct {
seat string
listRead bool
want string
}{
{"mesh-dns-resolver", true, ""},
{"mesh-dns-resolver", false, ""},
{"mesh-broker", true, "held once for the mesh, is held on ace and novox"},
{"mesh-broker", false, "held once for the mesh, is held on ace and novox"},
{"a-seat-not-listed", false, "the seat list is not read yet, so whether the seat a-seat-not-listed may be held on ace and novox waits for it"},
{"a-seat-not-listed", true, "held once for the mesh, is held on ace and novox"},
} {
a := seat.Assignments{Node: "ace", Modules: []seat.PlacedModule{{Module: "dns"}}, Holds: []seat.Holding{{Claim: c.seat, Scope: "mesh", Module: "dns"}}}
b := a
b.Node = "novox"
r := CheckAssignments(map[string]seat.Assignments{"ace": a, "novox": b}, known, enrolled, c.listRead)
got := strings.Join(r, "; ")
if (c.want == "") != (len(r) == 0) || !strings.Contains(got, c.want) || (!c.listRead && c.seat == "a-seat-not-listed" && strings.Contains(got, "held once")) {
t.Errorf("%s on two nodes, the list read %v: refusals %v, want %q", c.seat, c.listRead, r, c.want)
}
}
clash := g14
clash.Modules = append([]seat.PlacedModule{{Module: "ghost", Ports: map[string]int64{"80": 18080}}}, g14.Modules...)
clash.Modules[2] = seat.PlacedModule{Module: "app", Ports: map[string]int64{"8080": 18080}, Bindings: map[string]seat.Provider{"postgres": {Node: "ace", Module: "postgres"}}}
clash.Pins = map[string]seat.Provider{"shell": {Node: "g14", Module: "bash"}}
// ADR 0564 clause 6, Limits 5 (issue 1952 S8): mesh-watcher and mesh-self-check are never placed on one node.
clash.Modules = append(clash.Modules, seat.PlacedModule{Module: "mesh-watcher"}, seat.PlacedModule{Module: "mesh-self-check"})
twice := novox
twice.Node = "stranger"
twice.Decided.Address = "10.77.0.2"
r := CheckAssignments(map[string]seat.Assignments{"g14": clash, "novox": novox, "stranger": twice}, known, enrolled, true)
for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080",
"bound to ace/postgres", "pin of shell names g14/bash", "stranger is a node with no record",
"mesh-store, held once for the mesh, is held on novox and stranger",
"g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it",
"g14: mesh-watcher and mesh-self-check are never placed on one node (ADR 0227 §6, ADR 0564 clause 6)"} {
if !slices.ContainsFunc(r, func(line string) bool { return strings.Contains(line, want) }) {
t.Errorf("no refusal says %q; got %v", want, r)
}
}
})
t.Run("changes laid over the keys", func(t *testing.T) {
laid, findings := LayChanges(current, []seat.ProposalChange{
{Op: client.OpPlace, Node: "g14", Module: "new"},
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
{Op: client.OpUnpin, Node: "g14", Provision: "shell"},
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
})
if len(findings) != 0 || len(laid) != 2 || !reflect.DeepEqual(client.Assigned(laid["g14"]), []string{"app", "new"}) ||
len(laid["g14"].Holds) != 0 || laid["g14"].Pins != nil || laid["novox"].Decided.Site != "attic" || !laid["novox"].Decided.Hub {
t.Fatalf("laid %+v, %v", laid, findings)
}
if !reflect.DeepEqual(client.Assigned(current["g14"]), []string{"zsh", "app"}) {
t.Error("laying changed the current assignments")
}
// Design 47 line 1051 (the Choices table: a module's watched branch and its build in the placement key, set by a
// proposal): a place change sets them, and a re-place naming none keeps the module's.
onBeta, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta", Build: "b1"}})
again, g := LayChanges(onBeta, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Ports: map[string]int64{"80": 8081}}})
if m := onBeta["g14"].Modules[0]; len(f) != 0 || m.Branch != "beta" || m.Build != "b1" {
t.Errorf("a place change's branch and build: %+v, %v", m, f)
}
if m := again["g14"].Modules[0]; len(g) != 0 || m.Branch != "beta" || m.Build != "b1" || m.Ports["80"] != 8081 {
t.Errorf("a re-place dropped the branch and build: %+v, %v", m, g)
}
if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta"}}}) ==
ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) {
t.Error("an ask bound to a place change's digest does not bind its branch")
}
if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Build: "b1"}}}) ==
ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) {
t.Error("an ask bound to a place change's digest does not bind its build (design 47 G 360)")
}
if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" {
t.Errorf("a node with no key: %+v, %v", fresh, f)
}
for name, c := range map[string]struct {
change seat.ProposalChange
want string
}{
"not placed": {seat.ProposalChange{Op: client.OpTakeOff, Node: "g14", Module: "nope"}, "not placed"},
"no such field": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "colour", Value: "x"}, "no decided field"},
"wrong type": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "hub", Value: "x"}, "does not take"},
"unknown op": {seat.ProposalChange{Op: "move", Node: "g14"}, "no change of assignments"},
"pin of nobody": {seat.ProposalChange{Op: client.OpPin, Node: "g14", Provision: "x"}, "provider"},
"not held": {seat.ProposalChange{Op: client.OpUnhold, Node: "g14", Module: "app", Claim: "x"}, "holds no"},
"bad port": {seat.ProposalChange{Op: client.OpPlace, Node: "g14", Module: "x", Ports: map[string]int64{"a": 1}}, "port"},
"seed of another": {seat.ProposalChange{Op: client.OpSeed, Node: "g14", Whole: &novox}, "seed"},
} {
if _, f := LayChanges(current, []seat.ProposalChange{c.change}); len(f) != 1 || !strings.Contains(f[0], c.want) {
t.Errorf("%s: %v, want a finding naming %q", name, f, c.want)
}
}
})
t.Run("a proposal, its digests and its refusals", func(t *testing.T) {
if got := NewProposalID(time.Date(2026, 10, 28, 18, 30, 0, 0, time.UTC)); !strings.HasPrefix(got, "p-20261028t183000-") || client.CheckProposalID(got) != nil {
t.Errorf("NewProposalID = %q", got)
}
p := seat.Proposal{ID: id, By: "operator@anchor", Why: "move the app", At: time.Unix(1, 0).UTC(),
Changes: []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, Base: map[string]int64{"g14": 4},
State: client.ProposalAnswering}
raw, err := EncodeProposal(p)
if err != nil {
t.Fatal(err)
}
if back, err := DecodeProposal(raw); err != nil || back.ID != id || back.Base["g14"] != 4 || !reflect.DeepEqual(Nodes(back), []string{"g14"}) {
t.Fatalf("round trip %+v, %v", back, err)
}
if _, err := DecodeProposal([]byte(`{"v":9}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("version 9: %v", err)
}
seed := p
seed.Seed, seed.ExportDigest = true, ExportDigest(current)
seed.Changes = []seat.ProposalChange{{Op: client.OpSeed, Node: "g14", Whole: &g14}}
if _, err := EncodeProposal(seed); err != nil {
t.Errorf("the seed: %v", err)
}
back := p
back.HandBack, back.Changes = true, nil
if _, err := EncodeProposal(back); err != nil {
t.Errorf("a hand-back: %v", err)
}
// The digests are what the earlier holder's code made of the same values (issue 1966: a proposal's ask binds its
// changes' digest, and a seed its export's), whatever order the generated types keep their fields in.
rich := seat.Proposal{ID: id, By: "operator@anchor", Why: "w", At: time.Unix(1, 0).UTC(), State: client.ProposalAnswering,
Base: map[string]int64{"g14": 4, "lab": 0, "novox": 2},
Changes: []seat.ProposalChange{
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
{Op: client.OpPlace, Node: "g14", Module: "new", Ports: map[string]int64{"80": 8080}, Bindings: map[string]seat.Provider{"pg": {Node: "novox", Module: "postgres"}}},
{Op: client.OpPin, Node: "g14", Provision: "shell", Provider: &seat.Provider{Node: "g14", Module: "bash"}},
{Op: client.OpHold, Node: "g14", Module: "app", Claim: "node-shell", Scope: "node", Kind: "k", Site: "s"},
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
{Op: client.OpDecide, Node: "novox", Field: "peer-names", Value: map[string]any{"k": "v"}},
{Op: client.OpConfig, Node: "g14", Module: "app", MeshWide: map[string]any{"a": float64(1)}, Layer: map[string]any{"b": "c"}, MeshWideValues: map[string]any{"d": true}, Values: map[string]any{"e": []any{"f"}}},
{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7", Account: "x", AgentAccountHome: "/h"}, Keys: &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "se"}},
}}
for name, c := range map[string]struct{ got, want string }{
"export": {ExportDigest(current), "sha256:742f03d8790d7efa6abff76636f4a69ba2704d646233960f52ce08604b96e128"},
"rich changes": {ChangesDigest(rich), "sha256:056338a2143d8c1af4e1275f41ffd06f18f3af3396a196d4444df83a0434475c"},
"take-off": {ChangesDigest(p), "sha256:7ce48744dea7934e6f3a6a1eeffb28df4096ee6a6a6b4ae29e15fb019044d8de"},
"seed": {ChangesDigest(seed), "sha256:731fe40d188acdf4a27ea73f3a5f1f9d584c3a8af7eaf15d4fb18e5da4853515"},
"no changes": {ChangesDigest(seat.Proposal{ID: id, Base: map[string]int64{"g14": 4}}), "sha256:3c5f1509c2d1c867954c47303e8b8d84bd4cd9aa3458464c21933c6496999bee"},
} {
if c.got != c.want {
t.Errorf("the digest of %s is %s, and the earlier holder made %s", name, c.got, c.want)
}
}
if ExportDigest(current) != ExportDigest(map[string]seat.Assignments{"novox": novox, "g14": g14}) || ExportDigest(current) == ExportDigest(nil) {
t.Error("the export digest depends on order or not on the assignments")
}
// 774/S42/AC1 (ADR 0587 §7): an add-node carries its enrolment's warrant naming its node and address, as its one
// change, from the enroller alone; a warrant naming another node or address is refused.
// At the operator's yes it carries the tunnel key alone (the operator, 2026-11-02 01:05).
joining := seat.ProposalChange{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7"},
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}
add := seat.Proposal{ID: id, By: "ace.mesh-enrolment", Why: "lab joins", At: time.Unix(1, 0).UTC(),
Changes: []seat.ProposalChange{joining}, Base: map[string]int64{"lab": 0}, State: client.ProposalAnswering,
Warrant: &seat.Warrant{Node: "lab", Address: "10.0.0.7", Ask: "a-1", Warrant: "w-1"}}
if raw, err := EncodeProposal(add); err != nil {
t.Errorf("an add-node: %v", err)
} else if back, err := DecodeProposal(raw); err != nil || back.Warrant == nil || back.Changes[0].Keys.Overlay != "wg" {
t.Errorf("an add-node round trip: %+v, %v", back, err)
}
if AddNodeRefusal(add, enroller, enrollers) != "" || !strings.Contains(AddNodeRefusal(add, "mesh-delivery-assigner", enrollers), "enroll-lab's alone") {
t.Errorf("add-node by caller: %q, %q", AddNodeRefusal(add, enroller, enrollers), AddNodeRefusal(add, "mesh-delivery-assigner", enrollers))
}
if laid, f := LayChanges(current, add.Changes); len(f) != 0 || laid["lab"].Decided.Address != "10.0.0.7" || len(laid["lab"].Modules) != 0 {
t.Errorf("an add-node laid: %+v, %v", laid, f)
}
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpAddNode, Node: "g14", Decided: &seat.NodeDecided{Address: "10.0.0.9"},
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}}); len(f) != 1 || !strings.Contains(f[0], "placed already") {
t.Errorf("an add-node of a placed node: %v", f)
}
for name, mut := range map[string]func(*seat.Proposal){
"warrant of another node": func(p *seat.Proposal) {
p.Warrant = &seat.Warrant{Node: "pi", Address: "10.0.0.7", Ask: "a", Warrant: "w"}
},
"warrant of another address": func(p *seat.Proposal) {
p.Warrant = &seat.Warrant{Node: "lab", Address: "10.0.0.8", Ask: "a", Warrant: "w"}
},
"no warrant": func(p *seat.Proposal) { p.Warrant = nil },
"no tunnel key": func(p *seat.Proposal) {
c := joining
c.Keys = &seat.ProposalChangeKeys{Identity: "id", Sealing: "seal"}
p.Changes = []seat.ProposalChange{c}
},
"half its identity keys": func(p *seat.Proposal) {
c := joining
c.Keys = &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id"}
p.Changes = []seat.ProposalChange{c}
},
"beside another change": func(p *seat.Proposal) {
p.Changes = append(p.Changes, seat.ProposalChange{Op: client.OpPlace, Node: "lab", Module: "zsh"})
},
"the hub": func(p *seat.Proposal) {
c := joining
c.Decided = &seat.NodeDecided{Address: "10.0.0.7", Hub: true}
p.Changes = []seat.ProposalChange{c}
},
"a warrant on a move": func(p *seat.Proposal) {
p.Changes, p.Base = []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, map[string]int64{"g14": 4}
},
} {
q := add
q.Changes = append([]seat.ProposalChange(nil), add.Changes...)
mut(&q)
if _, err := EncodeProposal(q); err == nil {
t.Errorf("an add-node with %s: taken", name)
}
}
// With its identity keys whole, as bound at redeem, it is taken too.
whole := add
whole.Changes = []seat.ProposalChange{joining}
whole.Changes[0].Keys = &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "seal"}
if _, err := EncodeProposal(whole); err != nil {
t.Errorf("an add-node with its identity keys: %v", err)
}
// An enrolment's end takes its never-confirmed node off again: the enroller's alone, of a node running nothing.
remove := add
remove.Changes = []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "lab"}}
if _, err := EncodeProposal(remove); err != nil || AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers) == "" ||
AddNodeRefusal(remove, enroller, enrollers) != "" || !slices.Equal(RemovedNodes(remove), []string{"lab"}) {
t.Errorf("a remove-node: %v, %q", err, AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers))
}
joined := map[string]seat.Assignments{"lab": {V: keys.Version, Node: "lab", Decided: seat.NodeDecided{Address: "10.0.0.7"}, Modules: []seat.PlacedModule{}}}
if laid, f := LayChanges(joined, remove.Changes); len(f) != 0 || len(laid) != 0 {
t.Errorf("a joining node removed: %v, %v", laid, f)
}
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "g14"}}); len(f) != 1 ||
!strings.Contains(f[0], "runs modules") {
t.Errorf("a running node removed: %v", f)
}
remove.Warrant = &seat.Warrant{Node: "pi", Address: "10.0.0.7", Ask: "a", Warrant: "w"}
if _, err := EncodeProposal(remove); err == nil {
t.Error("a remove-node on another node's warrant: taken")
}
for name, mut := range map[string]func(*seat.Proposal){
"no id": func(p *seat.Proposal) { p.ID = "" },
"no proposer": func(p *seat.Proposal) { p.By = "" },
"no why": func(p *seat.Proposal) { p.Why = " " },
"no change": func(p *seat.Proposal) { p.Changes = nil },
"no base": func(p *seat.Proposal) { p.Base = map[string]int64{"novox": 1} },
"no state": func(p *seat.Proposal) { p.State = "" },
"asked, no ask": func(p *seat.Proposal) { p.State = client.ProposalAsked },
"seed mixed": func(p *seat.Proposal) { p.Seed, p.ExportDigest = true, "d" },
"bad change": func(p *seat.Proposal) { p.Changes[0].Op = "swap" },
"seed, no digest": func(p *seat.Proposal) { *p = seed; p.ExportDigest = "" },
"hand-back with changes": func(p *seat.Proposal) { p.HandBack = true },
} {
q := p
q.Changes = append([]seat.ProposalChange(nil), p.Changes...)
mut(&q)
if _, err := EncodeProposal(q); err == nil {
t.Errorf("%s: taken", name)
}
}
})
// ADR 0571 clause 4: who holds which seat, from assignments; a node seat by its node.
t.Run("who holds which seat", func(t *testing.T) {
held := SeatHoldersOf(map[string]seat.Assignments{
"ace": {Node: "ace", Holds: []seat.Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}},
"novox": {Node: "novox", Holds: []seat.Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"},
{Claim: "photo-gallery", Scope: "mesh", Module: "photos"}}},
})
if o := held["photo-gallery"]; !o.Outside || len(o.Holders) != 1 || held["mesh-broker"].Outside {
t.Errorf("a claim outside the set is %+v", o)
}
keyed := map[string][]byte{}
for k, v := range held {
keyed[k], _ = json.Marshal(v)
}
if h, err := client.HoldersOf(keyed, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" {
t.Errorf("a replicated seat's holders are %v, %v", h, err)
}
if h, _ := client.HoldersOf(keyed, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 4 {
t.Errorf("a node seat's holder is %v, keys %v", h, held)
}
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); h != nil {
t.Errorf("a node seat nobody holds there is held by %v", h)
}
})
}
@@ -1,96 +0,0 @@
package assignment
import (
"encoding/json"
"fmt"
"strings"
client "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Assignments candidates (novox/hq ADR 0567 §3): for each node a proposal must hear from, the holder writes one key
// `<proposal>.<node>` in its state `candidates`: the node's assignments with the proposal laid over it for a node whose key
// it changes, or the node's current key for one whose composition it changes only through another node (Reached). The
// layouts are the seat's contract's (CandidatesByProposalNodeValue); what refuses a candidate is here.
// CandidateKey is the key of one node's candidate in one proposal: `<proposal>.<node>`, so a node watches `*.<node>` and a
// proposal's keys are `<proposal>.*`.
func CandidateKey(proposal, node string) (string, error) {
if err := client.CheckProposalID(proposal); err != nil {
return "", err
}
if err := keys.CheckName("node", node); err != nil {
return "", err
}
return seat.CandidatesByProposalNodeKey(proposal, node), nil
}
// ParseCandidateKey is the inverse of CandidateKey; it refuses any other key, naming it.
func ParseCandidateKey(key string) (proposal, node string, err error) {
proposal, node, ok := strings.Cut(key, ".")
if !ok || keys.CheckName("node", node) != nil || client.CheckProposalID(proposal) != nil {
return "", "", fmt.Errorf("assignments: %q is not an assignments candidate key, <proposal>.<node>", key)
}
return proposal, node, nil
}
func checkCandidate(c seat.CandidatesByProposalNodeValue) error {
if err := client.CheckProposalID(c.Proposal); err != nil {
return err
}
if c.Assignments != nil {
return client.CheckAssignments(*c.Assignments)
}
return nil
}
// EncodeCandidate is c as JSON at the keys' Version; it refuses what DecodeCandidate refuses.
func EncodeCandidate(c seat.CandidatesByProposalNodeValue) ([]byte, error) {
if err := checkCandidate(c); err != nil {
return nil, err
}
c.V = keys.Version
if c.Assignments != nil {
p := *c.Assignments
p.V = keys.Version
c.Assignments = &p
}
return json.Marshal(c)
}
// DecodeCandidate reads a candidate; it refuses an unknown version, a proposal id that is not one, and assignments the
// client refuses.
func DecodeCandidate(data []byte) (seat.CandidatesByProposalNodeValue, error) {
if err := keys.CheckVersion(data); err != nil {
return seat.CandidatesByProposalNodeValue{}, err
}
c, err := seat.DecodeCandidatesByProposalNodeValue(data)
if err != nil {
return seat.CandidatesByProposalNodeValue{}, fmt.Errorf("assignments: a candidate does not fit its shape: %w", err)
}
if err := checkCandidate(c); err != nil {
return seat.CandidatesByProposalNodeValue{}, err
}
return c, nil
}
// PulseAnswerKey is the key of the bound on one node's answer to a proposal: `answer.<proposal>.<node>`.
func PulseAnswerKey(proposal, node string) (string, error) {
if err := client.CheckProposalID(proposal); err != nil {
return "", err
}
if err := keys.CheckName("node", node); err != nil {
return "", err
}
return seat.PulsesByAnswerProposalNodeKey(proposal, node), nil
}
// PulseAskKey is the key of the bound on a proposal's raised ask: `ask.<proposal>`.
func PulseAskKey(proposal string) (string, error) {
if err := client.CheckProposalID(proposal); err != nil {
return "", err
}
return seat.PulsesByAskProposalKey(proposal), nil
}
@@ -1,254 +0,0 @@
// Package assignments is the assigner's own check of assignments and of a proposal that changes it (novox/hq ADR 0567 §2,
// ADR 0634 §1 on where a seat's code goes): what the holder of the seat mesh-module-assigner decides before it writes
// anything. The layouts it works on are the seat's contract's (package meshmoduleassigner); what every reader of the
// assignments shares (the keys assembled into a node's assignments, the node key's digest, who holds a seat) is the seat's
// client's (package meshmoduleassignerclient). Nothing here is read by another module.
package assignment
import (
"encoding/json"
"fmt"
"sort"
"strings"
client "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// KeptApart are the modules never placed on one node, each row naming the record that asks it (novox/hq ADR 0564 clause 6
// and Limits 5; issue 1952 S8): no manifest word says it, so the check below holds the table. mesh-watcher hears the
// self-check's `checked` from a second machine (ADR 0227 §6), which on the self-check's own node would fall silent with
// it.
var KeptApart = []struct{ A, B, Record string }{
{A: "mesh-watcher", B: "mesh-self-check", Record: "ADR 0227 §6, ADR 0564 clause 6"},
}
// CheckAssignments is the holder's check of a proposal laid over every assignments (ADR 0567 §2), over the assignments by
// node, the modules that are known and the nodes with a record. It returns every refusal, in node order, never only the
// first: assignments naming a node not enrolled or a module that is not known (held by the indexer, or provided by the
// mesh itself: the caller's `known` says both); two modules of one node on the same host port; a pin, binding or holding
// naming a provider that is not placed where it says; a seat of scope mesh held twice, unless the seat is replicated
// (catalogue.Seat's Replicated, ADR 0223), which is held once per node; two nodes with one address on the private
// network; more than one hub; two modules KeptApart on one node. A host port a manifest declares without a number and a
// requirement left unbound are the composition's to find (its ForeseenPorts and Unbound).
//
// **A seat the list does not name is not judged held once while the list is unread** (novox/hq ADR 0636 §1, issue 1966):
// listRead says the host has read the seat list from the seat mesh-catalog; before, catalogue holds the bootstrapped
// definitions alone, and a mesh seat it does not name, held on two nodes, may be a replicated one. Its refusal then says the
// judgement waits for the list (SeatListWaits), never that it is held twice; once the list is read, a seat it does not name
// is judged as a seat held once, as before.
func CheckAssignments(assignments map[string]seat.Assignments, known func(module string) bool, enrolled func(node string) bool, listRead bool) []string {
var refusals []string
nodes := sortedKeys(assignments)
placed := func(p seat.Provider) bool {
other, ok := assignments[p.Node]
if !ok {
return false
}
for _, m := range other.Modules {
if m.Module == p.Module {
return true
}
}
return false
}
meshSeats := map[string]string{}
addresses := map[string]string{}
var hubs []string
for _, node := range nodes {
p := assignments[node]
if p.Node != node {
refusals = append(refusals, fmt.Sprintf("the assignments of %s names the node %q", node, p.Node))
}
if a := strings.TrimSpace(p.Decided.Address); a != "" {
if other, twice := addresses[a]; twice {
refusals = append(refusals, fmt.Sprintf("%s and %s both have the address %s on the private network", other, node, a))
}
addresses[a] = node
}
if p.Decided.Hub {
hubs = append(hubs, node)
}
if !enrolled(node) {
refusals = append(refusals, fmt.Sprintf("%s is a node with no record", node))
}
placedHere := map[string]bool{}
for _, m := range p.Modules {
placedHere[m.Module] = true
}
for _, apart := range KeptApart {
if placedHere[apart.A] && placedHere[apart.B] {
refusals = append(refusals, fmt.Sprintf("%s: %s and %s are never placed on one node (%s)", node, apart.A, apart.B, apart.Record))
}
}
hostPorts := map[int64]string{}
for _, m := range p.Modules {
if !known(m.Module) {
refusals = append(refusals, fmt.Sprintf("%s: the indexer holds no module %s", node, m.Module))
}
for _, port := range sortedKeys(m.Ports) {
at := m.Ports[port]
if other, clash := hostPorts[at]; clash && other != m.Module {
refusals = append(refusals, fmt.Sprintf("%s: %s and %s are both published on host port %d", node, other, m.Module, at))
}
hostPorts[at] = m.Module
}
for _, provision := range sortedKeys(m.Bindings) {
if to := m.Bindings[provision]; !placed(to) {
refusals = append(refusals, fmt.Sprintf("%s: %s's %s is bound to %s/%s, which is not placed there", node, m.Module, provision, to.Node, to.Module))
}
}
}
for _, provision := range sortedKeys(p.Pins) {
if to := p.Pins[provision]; !placed(to) {
refusals = append(refusals, fmt.Sprintf("%s: the pin of %s names %s/%s, which is not placed there", node, provision, to.Node, to.Module))
}
}
for _, h := range p.Holds {
if h.Scope != "mesh" {
continue
}
name := h.Claim + "/" + h.Kind
s, defined := catalogue.SeatNamed(h.Claim)
if defined && s.Replicated {
// A replicated mesh seat (ADR 0223) is held on several nodes at once, each answering the same thing;
// it is still held once on each node.
name += "/" + node
}
if other, twice := meshSeats[name]; twice {
if !defined && !listRead {
refusals = append(refusals, fmt.Sprintf(SeatListWaits, h.Claim, other, node))
} else {
refusals = append(refusals, fmt.Sprintf("the seat %s, held once for the mesh, is held on %s and %s", h.Claim, other, node))
}
}
meshSeats[name] = node
}
}
if len(hubs) > 1 {
refusals = append(refusals, fmt.Sprintf("the private network has one hub, and %s are each placed as it", strings.Join(hubs, ", ")))
}
return refusals
}
// SeatListWaits is the refusal of a mesh seat held on two nodes that the unread seat list does not name (ADR 0636 §1): the
// seat, and the two nodes.
const SeatListWaits = "the seat list is not read yet, so whether the seat %s may be held on %s and %s waits for it: " +
"the assigner judges it once it has read the list from the seat mesh-catalog (novox/hq ADR 0636 §1)"
// WaitsForSeatList says a refusal is SeatListWaits': it ends nothing, and is judged again once the list is read.
func WaitsForSeatList(refusal string) bool {
return strings.HasPrefix(refusal, SeatListWaits[:strings.Index(SeatListWaits, "%")])
}
// Change is what `assignments-changed` says of one node (ADR 0567 §6): the node, the revision of its key before (0: it had
// none) and after (0: the key was deleted), and its modules added, removed or changed against the assignments before.
type Change struct {
Node string `json:"node"`
Before uint64 `json:"before"`
After uint64 `json:"after"`
Added []string `json:"added,omitempty"`
Removed []string `json:"removed,omitempty"`
Changed []string `json:"changed,omitempty"`
}
// Body is the change as the seat's event carries it, one entry of its `nodes`.
func (c Change) Body() map[string]any {
raw, _ := json.Marshal(c)
var out map[string]any
_ = json.Unmarshal(raw, &out)
return out
}
// ChangeOf is what changed from before to after on one node; before is the zero Assignments for a node new to assignments.
// A change of the node's pins or holdings counts as a change of every module placed on it both times, since the
// composition of each may read them.
func ChangeOf(before, after seat.Assignments) Change {
c := Change{Node: after.Node}
if c.Node == "" {
c.Node = before.Node
}
was := map[string]seat.PlacedModule{}
for _, m := range before.Modules {
was[m.Module] = m
}
nodeWide := !jsonEqual(before.Pins, after.Pins) || !jsonEqual(before.Holds, after.Holds)
now := map[string]bool{}
for _, m := range after.Modules {
now[m.Module] = true
old, ok := was[m.Module]
switch {
case !ok:
c.Added = append(c.Added, m.Module)
case nodeWide || !jsonEqual(old, m):
c.Changed = append(c.Changed, m.Module)
}
}
for _, m := range before.Modules {
if !now[m.Module] {
c.Removed = append(c.Removed, m.Module)
}
}
sort.Strings(c.Added)
sort.Strings(c.Removed)
sort.Strings(c.Changed)
return c
}
// SortChanges orders assignments-changed event's nodes by node.
func SortChanges(c []Change) {
sort.Slice(c, func(i, j int) bool { return c[i].Node < c[j].Node })
}
func jsonEqual(a, b any) bool {
ja, _ := json.Marshal(a)
jb, _ := json.Marshal(b)
return string(ja) == string(jb)
}
// SeatHoldersOf is the state composed from every node's assignments (ADR 0567 §1): each holding by the seat it claims,
// keyed as the client's SeatHoldersKey says, with the module holding it and its node. A claim naming a seat's former name
// is keyed by the seat's name (ADR 0122); one naming no seat is kept, marked Outside. Deterministic, so a writer writes
// only a key that differs.
func SeatHoldersOf(assignments map[string]seat.Assignments) map[string]seat.SeatHolders {
out := map[string]seat.SeatHolders{}
for _, n := range sortedKeys(assignments) {
for _, h := range assignments[n].Holds {
node := ""
if h.Scope == "node" {
node = n
}
s, known := catalogue.SeatNamed(h.Claim)
name := h.Claim
if known {
name = s.Name
}
key, err := client.SeatHoldersKey(name, node)
if err != nil {
continue
}
sh := out[key]
sh.V, sh.Seat, sh.Scope, sh.Node, sh.Outside = keys.Version, name, h.Scope, node, !known
sh.Holders = append(sh.Holders, seat.SeatHoldersHoldersItem{Module: h.Module, Node: n})
out[key] = sh
}
}
for k, sh := range out {
sort.Slice(sh.Holders, func(i, j int) bool {
return sh.Holders[i].Node+"/"+sh.Holders[i].Module < sh.Holders[j].Node+"/"+sh.Holders[j].Module
})
out[k] = sh
}
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}

Some files were not shown because too many files have changed in this diff Show More