Compare commits

..
Author SHA1 Message Date
jschoubben 6efedf13c0 mesh-grants: ask the controller's root-free verb and wake on its condition events, so a verified sender is composed (issue 1945)
The part composed every membership with no root-free input, so telegram's verified-sender was dropped and the membership held as a shadow
difference. It now asks mesh-controller.root-free for the machines a composition needs, per pass, failing closed and said once, and wakes
on root-not-free condition-raised and condition-cleared.
2026-10-26 11:10:37 +01:00
1901 changed files with 16796 additions and 282677 deletions
-9
View File
@@ -1,10 +1 @@
vendor/
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
modules/*/cmd/*/mesh-*
!modules/*/cmd/*/mesh-*.go
# `go build .` at a module's root leaves its program there too (the view's, met a second time).
modules/*/mesh-*
# The delivery SDK's commands (design 47 G §464), built in their own directory or at the SDK's root.
/modules/mesh-delivery-sdk/cmd/seatgen/seatgen
/modules/mesh-delivery-sdk/seatgen/seatgen
/modules/mesh-delivery-sdk/mesh-pack
-293
View File
@@ -1,293 +0,0 @@
// Package asks holds the layout of the assigner's state `asks`: the keys of the asks the assigner puts to a person and the
// value it keeps at each, which the assigner writes and the view and the teller read (novox/hq ADR 0634 §1, ADR 0635 §5). It
// is the assigner's own published package. The gate's merge ask (`merge.<group>.<heads hash>`) needs proof of who answered
// (ADR 0421 §1), so it is answered on the phone alone until issue 953 closes. The assigner's asks need no such proof, and the
// view may answer them: the hold of a fingerprint whose health check failed (`hold.<module>.<fingerprint hex>`: release it or
// keep it held), the restore of a data module set back (`restore.<node>.<module>.<fingerprint hex>`: restore and lose newer
// writes, or leave it stopped, ADR 0429 §6), a backup that failed before an apply, a move a person must allow for a module of
// policy record, and the authorising ask of a proposal of assignments.
package asks
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/kept"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const (
// AskKindHold is the kind of the assigner's ask about a held fingerprint.
AskKindHold = "hold"
// AskKindRestore is the kind of the assigner's ask about a data module set back.
AskKindRestore = "restore"
// AskKindBackup is the kind of the assigner's ask about a backup before an apply that failed: back up again, or go on
// without one (novox/hq ADR 0539 clause 16).
AskKindBackup = "backup"
// AskKindRecord is the kind of the assigner's ask about a move a person must allow, for a module of policy record: a
// composition judged to recreate a container or move data, or one that cannot be judged (ADR 0539 clause 14).
AskKindRecord = "record"
// AskKindAssignments is the kind of the authorising ask about a proposal of assignments (novox/hq ADR 0567 §5):
// `assignments.<proposal id>`. The kind needs no proof, as the bus's put does not: its yes is an Approve, which only a
// channel proving who answered takes and the view never does (ViewAnswerFits).
AskKindAssignments = "assignments"
// AskKindOutside is the kind of the assigner's ask about a moved module whose failing health check has waited past its bound
// on an outside cause (design 47 G §706): go on, or count it as failed. `outside.<node>.<module>.<fingerprint hex>`.
AskKindOutside = "outside"
// AskKindRevert is the kind of the assigner's ask about a held revert, one with no restore point (design 47 G §710): keep it
// held, or start the reverted build on the live data. `revert.<node>.<module>.<fingerprint hex>`, the reverted build's.
AskKindRevert = "revert"
)
// AskKey is an ask's key read whole: its Kind, and the parts its kind has: a merge's Group and heads Hash, a hold's Module
// and fingerprint FP, a restore's, a backup's, an outside's and a revert's Node, Module and FP, a record's Node, Module and the digest Composed of the
// composition it would allow, the assignments' Proposal.
type AskKey struct {
Proposal string
Kind string
Group string
Hash string
Node string
Module string
FP string
Composed string
}
// AskNeedsProof says whether an ask of this kind must prove who answered it, so the view may not answer it: the merge's
// Approve does (ADR 0421 §1, issue 953), the assigner's hold and restore do not. An unknown kind is refused.
func AskNeedsProof(kind string) (bool, error) {
switch kind {
case meshdeliveryclient.AskKindMerge:
return true, nil
case AskKindHold, AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert:
return false, nil
}
return false, fmt.Errorf("asks: %q is not a kind of ask, %s, %s, %s, %s, %s, %s, %s or %s", kind, meshdeliveryclient.AskKindMerge, AskKindHold,
AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert)
}
// HoldAskKey is the key of the ask about a module's held fingerprint: `hold.<module>.<hex>`.
func HoldAskKey(module, fingerprint string) (string, error) {
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindHold + "." + key, nil
}
// RestoreAskKey is the key of the ask about a data module set back on a node from a fingerprint:
// `restore.<node>.<module>.<hex>`.
func RestoreAskKey(node, module, fingerprint string) (string, error) {
if err := keys.CheckName("node", node); err != nil {
return "", err
}
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindRestore + "." + node + "." + key, nil
}
// OutsideAskKey is the key of the ask about a module's failing check on a node that waited past its bound on an outside cause:
// `outside.<node>.<module>.<hex>`.
func OutsideAskKey(node, module, fingerprint string) (string, error) {
key, err := RestoreAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindOutside + strings.TrimPrefix(key, AskKindRestore), nil
}
// OutsideAbout is what the outside ask is about: `<node>.<module>.<hex>` (design 47 G §706).
func OutsideAbout(key string) string { return strings.TrimPrefix(key, AskKindOutside+".") }
// BackupAskKey is the key of the ask about a module's failed backup on a node before it applies a fingerprint:
// `backup.<node>.<module>.<hex>`.
func BackupAskKey(node, module, fingerprint string) (string, error) {
part, err := catalogkey.KeyPart(fingerprint)
if err != nil {
return "", err
}
key, err := nodeengineclient.ComposedKey(node, module, part)
if err != nil {
return "", err
}
return AskKindBackup + "." + key, nil
}
// RevertAskKey is the key of the ask about a held revert of a module on a node to the reverted build's fingerprint:
// `revert.<node>.<module>.<hex>` (design 47 G §710).
func RevertAskKey(node, module, fingerprint string) (string, error) {
key, err := BackupAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindRevert + strings.TrimPrefix(key, AskKindBackup), nil
}
// AssignmentsAskKey is the key of the authorising ask about a proposal of assignments: `assignments.<proposal id>` (ADR 0567 §5).
func AssignmentsAskKey(proposal string) (string, error) {
if err := meshmoduleassignerclient.CheckProposalID(proposal); err != nil {
return "", err
}
return AskKindAssignments + "." + proposal, nil
}
// RecordAskKey is the key of the ask about a move of a module on a node to the composition whose digest it names:
// `record.<node>.<module>.<digest hex>`, so a new revision at the same fingerprint is asked again (ADR 0539 clause 14).
func RecordAskKey(node, module, composed string) (string, error) {
have, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return "", err
}
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return "", err
}
return AskKindRecord + "." + have + "." + strings.TrimPrefix(composed, "sha256:"), nil
}
// ParseAnyAskKey reads an ask key of any kind (a merge, hold, restore, backup, record, assignments, outside or revert ask); it refuses an unknown
// kind and a key whose parts do not fit its kind, naming the key.
func ParseAnyAskKey(key string) (AskKey, error) {
kind, rest, _ := strings.Cut(key, ".")
refused := func(err error) (AskKey, error) {
return AskKey{}, fmt.Errorf("asks: %q is not an ask key: %w", key, err)
}
switch kind {
case meshdeliveryclient.AskKindMerge:
_, group, hash, err := meshdeliveryclient.ParseAskKey(key)
if err != nil {
return AskKey{}, err
}
return AskKey{Kind: kind, Group: group, Hash: hash}, nil
case AskKindHold:
module, fp, err := builds.ParseBuilderBuildKey(rest)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Module: module, FP: fp}, nil
case AskKindRestore:
node, build, ok := strings.Cut(rest, ".")
if !ok {
return refused(fmt.Errorf("a restore ask is restore.<node>.<module>.<fingerprint hex>"))
}
if err := keys.CheckName("node", node); err != nil {
return refused(err)
}
module, fp, err := builds.ParseBuilderBuildKey(build)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindOutside:
k, err := ParseAnyAskKey(AskKindRestore + "." + rest)
if err != nil {
return refused(fmt.Errorf("an outside ask is outside.<node>.<module>.<fingerprint hex>"))
}
k.Kind = kind
return k, nil
case AskKindBackup, AskKindRevert:
node, module, part, isNode, err := nodeengineclient.ParseComposedKey(rest)
if err != nil || isNode {
return refused(fmt.Errorf("a %s ask is %s.<node>.<module>.<fingerprint hex>", kind, kind))
}
fp, err := catalogkey.ParseKeyPart(part)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindAssignments:
if err := meshmoduleassignerclient.CheckProposalID(rest); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Proposal: rest}, nil
case AskKindRecord:
parts := strings.Split(rest, ".")
if len(parts) != 3 {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
node, module, isNode, err := nodeengineclient.ParseHaveKey(parts[0] + "." + parts[1])
if err != nil || isNode {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
composed := "sha256:" + parts[2]
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, Composed: composed}, nil
}
_, err := AskNeedsProof(kind)
return refused(err)
}
// Ask is the value of a key in the assigner's state asks: the router's ask whole, as the assigner composes it, so the teller
// publishes it unchanged. Asking counts the times the assigner wrote the ask at this key, as the gate's ask does: 0 or 1 for
// the first, n for the n-th, and the ask's id is meshdeliveryclient.AskingID(key, Asking), so each asking has its own id and
// an answer to an earlier asking is never taken for a later one. The ask is the operator-channel seat's as its JSON: its id,
// its options and its digest are checked by the assigner and the teller through operator-channel's client (novox/hq ADR
// 0627 §4).
type Ask struct {
V int `json:"v"`
Ask json.RawMessage `json:"ask"`
Asking int `json:"asking,omitempty"`
}
func (a Ask) check(key string) error {
ask, err := ParseAnyAskKey(key)
if err != nil {
return err
}
if proof, _ := AskNeedsProof(ask.Kind); proof {
return fmt.Errorf("asks: the assigner asks no %s ask: it needs proof of who answered", ask.Kind)
}
if a.Asking < 0 {
return fmt.Errorf("asks: the ask's asking %d is negative", a.Asking)
}
var fields map[string]json.RawMessage
if len(bytes.TrimSpace(a.Ask)) == 0 || json.Unmarshal(a.Ask, &fields) != nil || fields == nil {
return fmt.Errorf("asks: the value carries no router's ask")
}
return nil
}
// EncodeAsk is a, the ask at key, as JSON at keys.Version; it refuses a key of a kind that needs proof or of no kind, a
// negative asking, and a value carrying no router's ask.
func EncodeAsk(key string, a Ask) ([]byte, error) {
if err := a.check(key); err != nil {
return nil, err
}
a.V = keys.Version
return json.Marshal(a)
}
// DecodeAsk reads the ask at key; it refuses an unknown version (keys.ErrUnknownVersion) and what EncodeAsk refuses.
func DecodeAsk(key string, data []byte) (Ask, error) {
var a Ask
if err := kept.Decode(data, &a); err != nil {
return Ask{}, err
}
if err := a.check(key); err != nil {
return Ask{}, err
}
return a, nil
}
// SameAsk says whether two asks are the same value: the same version, asking and router's ask, its JSON compared compacted.
func SameAsk(a, b Ask) bool {
return a.V == b.V && a.Asking == b.Asking && bytes.Equal(compacted(a.Ask), compacted(b.Ask))
}
func compacted(raw json.RawMessage) []byte {
var b bytes.Buffer
if json.Compact(&b, raw) != nil {
return raw
}
return b.Bytes()
}
@@ -1,89 +0,0 @@
package asks
import (
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The keys equal what the assigner has always written (the expected strings are the trunk's own output for the same input),
// and each reads back to its parts.
func TestTheAssignersAskKeysKeepTheirFormsAndTheValueItsRules(t *testing.T) {
fp := "src2:" + strings.Repeat("ab", 32)
hex := strings.Repeat("ab", 32)
digest := "sha256:" + strings.Repeat("cd", 32)
hold, _ := HoldAskKey("gitea", fp)
restore, _ := RestoreAskKey("n1", "gitea", fp)
backup, _ := BackupAskKey("n1", "novox.be", fp)
assignments, _ := AssignmentsAskKey("p-20261028t183000-1a2b3c")
record, _ := RecordAskKey("n1", "gitea", digest)
revert, _ := RevertAskKey("n1", "nats", fp)
for key, want := range map[string]struct {
got string
back AskKey
}{
"hold": {hold, AskKey{Kind: "hold", Module: "gitea", FP: fp}},
"restore": {restore, AskKey{Kind: "restore", Node: "n1", Module: "gitea", FP: fp}},
"backup": {backup, AskKey{Kind: "backup", Node: "n1", Module: "novox.be", FP: fp}},
"assignments": {assignments, AskKey{Kind: "assignments", Proposal: "p-20261028t183000-1a2b3c"}},
"record": {record, AskKey{Kind: "record", Node: "n1", Module: "gitea", Composed: digest}},
"revert": {revert, AskKey{Kind: "revert", Node: "n1", Module: "nats", FP: fp}},
} {
back, err := ParseAnyAskKey(want.got)
if err != nil || back != want.back {
t.Errorf("%s: ParseAnyAskKey(%q) = %+v, %v; want %+v", key, want.got, back, err, want.back)
}
}
for got, want := range map[string]string{hold: "hold.gitea." + hex, restore: "restore.n1.gitea." + hex, backup: "backup.n1.novox=2ebe." + hex,
assignments: "assignments.p-20261028t183000-1a2b3c", record: "record.n1.gitea." + strings.Repeat("cd", 32), revert: "revert.n1.nats." + hex} {
if got != want {
t.Errorf("key %q, want %q", got, want)
}
}
merge, err := ParseAnyAskKey("merge.fix/1=2e2.d106c7950d28")
if err != nil || merge.Kind != "merge" || merge.Group != "fix/1.2" || merge.Hash != "d106c7950d28" {
t.Errorf("a merge ask key: %+v, %v", merge, err)
}
for _, bad := range []string{"", "other.x", "hold.gitea", "hold.gitea.src2:" + hex, "restore.n1", "backup.n1.node", "backup.n1.gitea." + hex[:60],
"assignment.AB", "record.n1.gitea." + hex[:10], "record.n1.node." + hex, "merge.g", "revert.n1.node", "revert.n1.nats." + hex[:60]} {
if _, err := ParseAnyAskKey(bad); err == nil {
t.Errorf("ParseAnyAskKey(%q) reads a wrong key", bad)
}
}
if proof, err := AskNeedsProof("merge"); err != nil || !proof {
t.Error("a merge ask needs proof")
}
for _, kind := range []string{"hold", "revert"} {
if proof, err := AskNeedsProof(kind); err != nil || proof {
t.Errorf("a %s ask needs no proof", kind)
}
}
if _, err := AskNeedsProof("view"); err == nil {
t.Error("an unknown kind of ask is taken")
}
// The value: the router's ask whole; an ask of a kind that needs proof, a negative asking and no ask are refused.
good := Ask{Ask: []byte(`{"id":"x"}`), Asking: 2}
data, err := EncodeAsk(hold, good)
if err != nil {
t.Fatalf("EncodeAsk: %v", err)
}
back, err := DecodeAsk(hold, data)
if err != nil || !SameAsk(back, Ask{V: keys.Version, Ask: []byte(` {"id": "x"} `), Asking: 2}) {
t.Errorf("DecodeAsk = %+v, %v", back, err)
}
for name, c := range map[string]struct {
key string
a Ask
}{"a merge ask": {"merge.g.d106c7950d28", good}, "a negative asking": {hold, Ask{Ask: good.Ask, Asking: -1}}, "no ask": {hold, Ask{}},
"a null ask": {hold, Ask{Ask: []byte("null")}}, "an unknown kind": {"other.x", good}} {
if _, err := EncodeAsk(c.key, c.a); err == nil {
t.Errorf("EncodeAsk takes %s", name)
}
}
if _, err := DecodeAsk(hold, []byte(`{"v":2,"ask":{}}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("DecodeAsk of another version = %v", err)
}
}
@@ -1,23 +0,0 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
)
require (
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
)
@@ -1,225 +0,0 @@
package main
import (
"context"
"maps"
"slices"
"strings"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The waits for an acting part's new version in the steps the assigner holds (design 47 G §266, §268, §276): an entry of the assignment
// graph whose acting edge names composing or placing waits, its want held, until the part's version runs on the edge's node. The wait is
// kept in the state `waits` under `acting.<node>.<module>.<part>`, said as `wait-held` when it starts and `wait-ended` (ran, released,
// replaced) when it ends; `assigner_waits` reads it and `assigner_release` with how `now` lets the entry move with the part's running
// version. A node away starts no wait: its clock starts when the node is present (§268).
// actingOf is the Module.Acting of module: the held acting edges of its entry on a node, those whose wait was not released left out.
func (p *passer) actingOf(module string, released map[string]bool) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
if p.engineOn(node) == module {
return nil // the node-engine's own want is held by no acting edge, so it is written first (§268, A-323, A-330)
}
for _, e := range p.entryEdges(node, module) {
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
}
}
return out
}
}
// engineOn is the module that holds the seat node-engine on node in the revision the pass computed, read from the acting edges'
// seat (A-330); "" when none is read.
func (p *passer) engineOn(node string) string {
if p.graph == nil {
return ""
}
return assigngraph.Engines(slices.Collect(maps.Values(p.graph.computed.Entries)))[node]
}
// entryEdges are the acting edges of the entry of module on node in the revision the pass computed.
func (p *passer) entryEdges(node, module string) []seat.Edge {
g := p.graph
if g == nil {
return nil
}
for _, ref := range g.computed.Revision.Entries {
if ref.Node == node && ref.Module == module {
var out []seat.Edge
for _, e := range g.computed.Entries[ref.Digest].Edges {
if e.Kind == "acting" {
out = append(out, e)
}
}
return out
}
}
return nil
}
// actingReleased is the waits released by hand and not yet run, by `<key>@<part's fingerprint>`.
func (p *passer) actingReleased() map[string]bool {
out := map[string]bool{}
for key, raw := range p.w.read(buckets.Waits()) {
if w, err := acting.Decode(raw); err == nil && w.Released && (acting.Is(key) || acting.IsOrder(key)) {
out[key+"@"+w.Fp] = true
}
}
return out
}
// actingWaits keeps the acting waits after the pass's wants: it starts the wait of each entry held for a part's version, ends the
// wait of one that ran, was replaced or was released, and says each.
func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]meshwants.Want) {
g := p.graph
if g == nil || p.kvs[buckets.Waits()] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits())
standing := map[string]bool{}
engines := assigngraph.Engines(slices.Collect(maps.Values(g.computed.Entries)))
held := map[string]bool{} // `<node>/<module>` of each entry held for a part's version
for _, ref := range g.computed.Revision.Entries {
entry := g.computed.Entries[ref.Digest]
if w, ok := wanted[ref.Module][ref.Node]; ok && w.Fp == entry.Fp {
continue // the entry's version is wanted already: nothing moves, so nothing waits
}
if engines[ref.Node] == ref.Module {
continue // the node-engine's own want waits for no acting part (§268, A-323, A-330)
}
for _, e := range entry.Edges {
if e.Kind != "acting" || !acting.Held(e.Step) {
continue
}
key := acting.Key(ref.Node, ref.Module, e.Module)
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
continue
}
standing[key] = true
held[ref.Node+"/"+ref.Module] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == entry.Fp && w.Fp == e.Fp {
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[ref.Node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: e.Step, Node: ref.Node, Module: ref.Module, Part: e.Module, PartNode: e.Node, Fp: e.Fp, Entry: entry.Fp,
Since: now, ClockFrom: now, Bound: acting.BoundOf(e).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.Fp+"."+entry.Fp); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", ref.Module, ref.Node, e.Module, err)
continue
}
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
}
}
p.actingCycles(g.computed.Entries, engines, held, n)
for key, raw := range kept {
if !acting.Is(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
how := acting.HowRan
if w.Released {
how = acting.HowReleased
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module())
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits(), key)
}
}
// actingReleaseSaid says wait-ended of a wait released by hand, once, and keeps its record so the entry is not held again.
func (p *passer) actingReleaseSaid(ctx context.Context, key string, w acting.Wait) {
if !w.Released || w.EndSaid {
return
}
body := acting.Body(w, assignment.Module())
body.How = acting.HowReleased
body.By = w.By
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+acting.HowReleased); err != nil {
p.logf("the release of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
return
}
w.EndSaid = true
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
}
// edgeOf is the edge a kept wait stands on, as an acting edge.
func (p *passer) edgeOf(w acting.Wait) (assign.ActingEdge, bool) {
return assign.ActingEdge{Node: w.PartNode, Part: w.Part, Fp: w.Fp, Step: w.Step}, true
}
// actingCycles writes the finding acting-cycle of each held want whose holding edges lie on a cycle no rule breaks, and clears it of
// every other want, a finding a pass before or another process wrote included (design 47 G A-323, A-330). An edge holds while its
// step is the assigner's, its part's version does not run and its wait was not released; the node-engine's own want is held by none.
func (p *passer) actingCycles(entries map[string]seat.AssignmentEntry, engines map[string]string, held map[string]bool, n assign.Nodes) {
if p.findings.Put == nil {
return
}
released := p.actingReleased()
cycles := assigngraph.HeldCycles(slices.Collect(maps.Values(entries)), func(e seat.AssignmentEntry, edge seat.Edge) bool {
return held[e.Node+"/"+e.Module] && engines[e.Node] != e.Module && edge.Kind == "acting" && acting.Held(edge.Step) &&
!released[acting.Key(e.Node, e.Module, edge.Module)+"@"+edge.Fp] && !n.Runs(assign.ActingEdge{Node: edge.Node, Part: edge.Module, Fp: edge.Fp, Step: edge.Step})
})
raised := map[string]bool{}
for _, at := range slices.Sorted(maps.Keys(cycles)) {
node, module, _ := strings.Cut(at, "/")
o := assign.ActingCycleFinding(node, module, cycles[at])
key := mcseatKey(o)
raised[key] = true
if p.said[key+"="+o.Summary] {
continue // written already as it stands
}
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", key, err)
continue
}
p.said[key+"="+o.Summary] = true
}
if p.findings.Keys == nil || p.findings.Delete == nil {
return
}
keys, err := p.findings.Keys()
if err != nil {
p.logf("the findings %s are not read, so none is cleared this pass: %v", assign.KindActingCycle, err)
return
}
for _, key := range keys {
if strings.HasSuffix(key, "."+assign.KindActingCycle) && !raised[key] {
if err := p.findings.Delete(key); err != nil {
p.logf("the finding %s is not cleared: %v", key, err)
continue
}
for said := range p.said {
if strings.HasPrefix(said, key+"=") {
delete(p.said, said)
}
}
}
}
}
@@ -1,184 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §266, §268, §276 (AS9): an entry with an acting edge in composing waits for the part's version to run on the edge's node;
// the wait is kept, said as wait-held with its bound and clock-from, read by assigner_waits, released by assigner_release with how
// `now` and ended when the version runs.
func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T) {
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, now)
var events []string
var bodies []meshwants.Wait
p.publishWants = func(event string, body any, id string) error {
events = append(events, event)
bodies = append(bodies, body.(meshwants.Wait))
return nil
}
entry := meshwants.AssignmentEntry{Node: "ace", Module: "nats", Fp: fpTest, Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Seat: "node-engine", Step: "compose", Bound: "15m"},
{Kind: "acting", Node: "ace", Module: "indexer", Fp: "src2:idx", Seat: "mesh-catalog", Step: "take-in"}}}
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
ctx := context.Background()
if got := p.actingOf("nats", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
t.Fatalf("the assigner holds composing only: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted) // said once
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("events %v", events)
}
b := bodies[0]
if b.Step != "compose" || b.Part != "mesh-host" || b.Fp != "src2:new" || b.Bound != "15m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(now) || b.Owner == "" {
t.Errorf("wait-held %+v", b)
}
key := acting.Key("ace", "nats", "mesh-host")
if _, err := acting.Decode(p.w.read(buckets.Waits())[key]); err != nil {
t.Fatalf("the wait is kept: %v", err)
}
// a release by hand lets the entry move, and ends the wait once
w, _ := acting.Decode(p.w.read(buckets.Waits())[key])
w.Released = true
raw, _ := acting.Encode(w)
p.w.put(buckets.Waits(), key, raw)
if got := p.actingOf("nats", p.actingReleased())("ace"); len(got) != 0 {
t.Errorf("a released wait still holds: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted)
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != acting.HowReleased {
t.Fatalf("events %v bodies %+v", events, bodies)
}
// the version runs: the record goes, said nothing more
n.Running["ace"]["mesh-host"] = nodeengine.Running{Fingerprint: "src2:new"}
p.actingWaits(ctx, n, wanted)
if len(p.w.read(buckets.Waits())) != 0 || len(events) != 2 {
t.Errorf("kept %v events %v", p.w.read(buckets.Waits()), events)
}
}
func TestAnActingWaitOfANodeAwayStartsNoClockAndAnEntryWantedAlreadyWaitsNot(t *testing.T) {
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
p.publishWants = func(string, any, string) error { t.Fatal("said"); return nil }
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": {Node: "ace", Module: "nats", Fp: "src2:a", Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Step: "compose"}}}}
n := assign.Nodes{Online: map[string]bool{}}
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{})
n.Online["ace"] = true
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:a"}}})
if len(p.w.read(buckets.Waits())) != 0 || strings.Contains(p.graph.computed.Entries["d1"].Fp, "x") {
t.Errorf("kept %v", p.w.read(buckets.Waits()))
}
}
func TestTheToolsReadAndReleaseAnActingWait(t *testing.T) {
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
w := acting.Wait{Step: "compose", Node: "ace", Module: "nats", Part: "mesh-host", PartNode: "ace", Fp: "src2:new", Entry: "src2:a", Since: time.Now(), ClockFrom: time.Now(), Bound: "30m0s"}
raw, _ := acting.Encode(w)
s.st.put(waitsState, acting.Key("ace", "nats", "mesh-host"), raw)
got, err := s.doWaits(map[string]any{"module": "nats"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).BoundEnds == "" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now"}, "x"); err == nil {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now", "why": "the new engine is known good"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
raw, _, _, _ = s.st.get(waitsState, acting.Key("ace", "nats", "mesh-host"))
if r, _ := acting.Decode(raw); !r.Released || !strings.Contains(r.By, "known good") {
t.Errorf("wait = %+v", r)
}
}
// Design 47 G §268, A-323, A-330: the assigner and the node-engine on a node are one step: the node-engine's own want waits for no
// acting part, so it moves first, and the node's other wants, the assigner's own included, wait for the new node-engine; a want held
// on a cycle no rule breaks is the finding acting-cycle (scope module, id <node>.<module>) naming each edge's step, cleared by the
// next pass that finds it on none.
func TestTheNodeEngineIsWantedFirstAndAWantHeldOnACycleIsAFinding(t *testing.T) {
p := newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Date(2026, 11, 13, 2, 0, 0, 0, time.UTC))
p.publishWants = func(string, any, string) error { return nil }
findings := map[string]mcseat.Finding{}
p.findings = assign.Findings{
Put: func(f mcseat.Finding) error { findings[f.Scope+"."+f.ID+"."+f.Kind] = f; return nil },
Delete: func(key string) error { delete(findings, key); return nil },
Keys: func() ([]string, error) {
var keys []string
for k := range findings {
keys = append(keys, k)
}
return keys, nil
}}
compose := meshwants.Edge{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:engine", Seat: "node-engine", Step: "compose"}
place := meshwants.Edge{Kind: "acting", Node: "ace", Module: "assigner", Fp: "src2:assigner", Seat: "mesh-wants", Step: "place"}
entries := map[string]meshwants.AssignmentEntry{
"d1": {Node: "ace", Module: "mesh-host", Fp: "src2:engine", Edges: []meshwants.Edge{place}},
"d2": {Node: "ace", Module: "assigner", Fp: "src2:assigner", Edges: []meshwants.Edge{compose}},
"d3": {Node: "ace", Module: "app", Fp: "src2:app", Edges: []meshwants.Edge{compose, place}},
}
p.graph = &graphKeeper{}
for d, e := range entries {
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: e.Node, Module: e.Module, Digest: d})
}
p.graph.computed.Entries = entries
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {
"mesh-host": {Fingerprint: "src2:old"}, "assigner": {Fingerprint: "src2:old"}}}}
wanted := map[string]map[string]mwc.Want{}
if got := p.actingOf("mesh-host", nil)("ace"); len(got) != 0 {
t.Errorf("the node-engine's want is held by %+v, want by no acting edge", got)
}
if got := p.actingOf("assigner", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
t.Errorf("the assigner's own want is held by %+v, want by the new node-engine", got)
}
p.actingWaits(context.Background(), n, wanted)
kept := p.w.read(buckets.Waits())
if kept[acting.Key("ace", "mesh-host", "assigner")] != nil || kept[acting.Key("ace", "assigner", "mesh-host")] == nil || kept[acting.Key("ace", "app", "mesh-host")] == nil {
t.Errorf("the waits kept: %v", kept)
}
if len(findings) != 0 {
t.Errorf("the cycle the node-engine's want breaks is a finding: %v", findings)
}
// Two parts holding each other across compose and place, neither the node-engine: no rule breaks it.
entries["d4"] = meshwants.AssignmentEntry{Node: "ace", Module: "x", Fp: "src2:x", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "y", Fp: "src2:y", Seat: "s-y", Step: "place"}}}
entries["d5"] = meshwants.AssignmentEntry{Node: "ace", Module: "y", Fp: "src2:y", Edges: []meshwants.Edge{{Kind: "acting", Node: "ace", Module: "x", Fp: "src2:x", Seat: "s-x", Step: "compose"}}}
p.graph.computed.Revision.Entries = append(p.graph.computed.Revision.Entries, meshwants.NodeEntryRef{Node: "ace", Module: "x", Digest: "d4"},
meshwants.NodeEntryRef{Node: "ace", Module: "y", Digest: "d5"})
p.actingWaits(context.Background(), n, wanted)
f, ok := findings["module.ace.x.acting-cycle"]
if len(findings) != 2 || !ok || !strings.Contains(f.Summary, "ace/x -[place]-> ace/y -[compose]-> ace/x") {
t.Fatalf("the findings of a cycle no rule breaks: %v", findings)
}
// x's new version runs: y no longer waits, so neither is held on a cycle, and both findings clear.
n.Running["ace"]["x"] = nodeengine.Running{Fingerprint: "src2:x"}
p.actingWaits(context.Background(), n, wanted)
if len(findings) != 0 {
t.Errorf("the findings after the cycle broke: %v", findings)
}
}
@@ -1,100 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askAgainTool asks one of the assigner's asks again by hand (issue 1966, ADR 0668 §5): the recovery of an ask that ended at
// the operator channel without an answer when the event that would have asked it again was not heard.
const askAgainTool = "assigner_ask_again"
// asksState and answersState are the assigner's asks and the teller's answers, as the runtime names them.
const (
asksState = "asks"
answersState = buckets.TellerModule + ".answers"
)
// askAgainByHand writes the ask at key again under a new asking, by the rules the teller's ask-ended writes it by (assign.AskedAgain:
// a proposal's authorising ask and a put's record ask are not), so the teller asks it again; an asking the operator answered
// is refused, naming the answer. The act is recorded through the seat mesh-hand-acts.
func askAgainByHand(st store, ask func(string, any) (json.RawMessage, error), key, why, by string, now time.Time) (any, error) {
switch {
case key == "":
return nil, errors.New("name the ask's key, as the tool asks lists it")
case why == "":
return nil, errors.New("a hand ask again says why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("a hand ask again names who asks it, and the runtime proved no caller")
}
current, _, found, err := st.get(asksState, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the assigner holds no ask under %s: the tool asks lists them", key)
}
_, routerAsk, err := assign.DecodeAsk(key, current)
if err != nil {
return nil, err
}
if raw, _, found, err := st.get(answersState, key); err == nil && found {
if a, err := meshdeliveryclient.DecodeAnswer(raw); err == nil && strings.HasPrefix(a.Warrant, routerAsk.ID+"@") {
return nil, fmt.Errorf("the asking %s was answered %q (%s): it is not asked again", routerAsk.ID, a.Option, a.Words)
}
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRecord {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
if raw, _, found, err := st.get(putsState, pkey); err == nil && found {
if p, err := puts.Decode(raw); err == nil {
put = &p
}
}
}
}
value, refused := assign.AskedAgain(tellerevents.Ended{Key: key, Ask: routerAsk.ID, Outcome: occ.OutcomeCancelled}, current, put, now)
if refused != "" {
return nil, errors.New(refused)
}
if err := st.put(asksState, key, value); err != nil {
return nil, fmt.Errorf("the ask %s is not written again: %w", key, err)
}
_, next, _ := assign.DecodeAsk(key, value)
taken, rerr := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askAgainTool + " " + key, Why: why + " (" + by + ")", Cause: "ask-again"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
}
return map[string]any{"key": key, "ended": routerAsk.ID, "asked": next.ID, "by": by, "hand-act": recorded}, nil
}
// askAgainTools is the tool over the runtime's states.
func askAgainTools(st store, ask func(string, any) (json.RawMessage, error), now func() time.Time) []stdio.Tool {
return []stdio.Tool{{Name: askAgainTool, Description: "Ask one of the assigner's asks of the operator again (issue 1966): the ask " +
"under key is written again under a new asking, which the teller asks on the operator channel, as when the channel says it " +
"ended without an answer. Refused for an asking the operator answered, a proposal's authorising ask (the seat's verb again " +
"asks it) and a record ask a put asked. Recorded as a hand act.",
Input: schema(map[string]string{"key": "the ask's key, as the tool asks lists it", "why": "why it is asked again"},
[]string{"key", "why"}),
Run: func(a map[string]any) (any, error) {
key, _ := a["key"].(string)
why, _ := a["why"].(string)
return askAgainByHand(st, ask, key, why, stdio.CallerOf(a), now())
}}}
}
@@ -1,239 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/process"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askEndedSubject is the teller's event ask-ended, as the manifest's consumes names it (`mesh-delivery-teller.ask-ended`).
var askEndedSubject = process.Account{Module: buckets.TellerModule}.EventSubject(tellerevents.AskEnded)
// staleOf says why the ask at a key is stale, as the pass reads it (passer.staleAsk), or "" when it is not or when the
// trunk is not read; nil reads nothing.
type staleOf func(key string) string
// staleOfWorld is staleOf over the world the pass reads: the same rule as the pass, over the same cache.
func staleOfWorld(w *world) staleOf {
p := &passer{w: w}
return func(key string) string {
trunk := w.trunk()
if len(trunk) == 0 {
return ""
}
return p.staleAsk(key, trunk)
}
}
// hearAskEnded hears the teller's ask-ended on the assigner's one hub of its durable consumer on EVENTS (registered before
// the hub starts): an ask of the assigner's that ended at the operator channel without an answer while its key stands is
// written again under a new asking, so the teller asks it again (issue 1966, note 151). asksKV is the assigner's asks; the
// puts are opened at each event, since a put's record ask is not written again (ADR 0546).
func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, logf func(string, ...any)) error {
return hub.Handle(askEndedSubject, func(m jetstream.Msg) {
var putsKV jetstream.KeyValue
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
if err := askAgain(ctx, asksKV, putsKV, stale, m.Data(), time.Now(), logf); err != nil {
logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
})
}
// channelAsksVerb is the operator-channel seat's verb asks, read at the assigner's consumer's bind (a seat verb: no module
// named).
const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks
// readEndedAtBind registers on the hub, before it starts, one read of the operator-channel seat's asks at each bind of the
// assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155,
// ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it
// (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start.
func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, logf func(string, ...any)) {
if asksKV == nil || bus == nil {
return
}
hub.OnBound(func(created time.Time) {
go func() {
if err := endedAtBind(ctx, js, asksKV, stale, bus, wait, created, time.Now(), logf); err != nil {
logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err)
}
}()
})
}
// callWait is how a kept call the assigner waits on is said (novox/hq ADR 0668 §3): heard says the callee's machine is
// heard, now is the clock, and findings the assigner's findings, where the call unanswered past
// meshconditionsclient.CallUnansweredAfter is the finding call-unanswered while heard, cleared when it is answered.
type callWait struct {
heard func() bool
now func() time.Time
findings assign.Findings
}
// asksCallID is the finding's id for the bind's read of the operator channel's asks.
const asksCallID = "operator-channel-asks"
// keep makes call again after every transport fault (busretry: a request that reached no one, or one whose deadline
// passed) until it is answered or ctx ends, each try itself a kept call; past the bound with the callee heard it writes
// the finding call-unanswered, and clears it when answered. No timer: each try is the wait.
func (cw callWait) keep(ctx context.Context, id, callee string, call func() (json.RawMessage, error), logf func(string, ...any)) (json.RawMessage, error) {
clock := cw.now
if clock == nil {
clock = time.Now
}
since, said := clock(), false
key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered)
for {
answer, err := call()
if err == nil || !(busretry.Undelivered(err) || busretry.TimedOut(err)) || ctx.Err() != nil {
if said && cw.findings.Delete != nil {
if derr := cw.findings.Delete(key); derr != nil {
logf("the finding %s is not cleared: %v", key, derr)
}
}
return answer, err
}
if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil {
continue
}
eerr := cw.findings.Put(assign.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since).Finding(clock()))
if eerr != nil {
logf("the finding %s is not written: %v", key, eerr)
continue
}
said = true
logf("%s is unanswered since %s and its callee's machine is heard: the finding %s", callee, since.UTC().Format(time.RFC3339), key)
}
}
// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and
// each ended asking written again.
func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
keys, err := asksKV.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil
}
if err != nil {
return err
}
held := map[string][]byte{}
for _, key := range keys {
if e, err := asksKV.Get(ctx, key); err == nil {
held[key] = e.Value()
}
}
query, ok := assign.ChannelAsksQuery(held)
if !ok {
return nil
}
answer, err := wait.keep(ctx, asksCallID, channelAsksVerb, func() (json.RawMessage, error) { return bus.CallKept(channelAsksVerb, query) }, logf)
if err != nil {
return err
}
ended, cut, err := assign.EndedAtChannel(held, answer)
if err != nil {
return err
}
if cut != "" {
logf("%s at the bind: %s", channelAsksVerb, cut)
}
var putsKV jetstream.KeyValue
if js != nil {
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
}
for _, e := range ended {
body, err := tellerevents.EncodeEnded(e)
if err == nil {
err = askAgain(ctx, asksKV, putsKV, stale, body, now, logf)
}
if err != nil {
return err
}
}
logf("%s at the bind of %s: %d of %d held asks ended at the channel without an answer", channelAsksVerb, created.Format(time.RFC3339), len(ended), len(held))
return nil
}
// askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the
// asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that
// could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped.
func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, stale staleOf, data []byte, now time.Time, logf func(string, ...any)) error {
e, err := tellerevents.DecodeEnded(data)
if err != nil {
logf("the teller's %s holds what is not an ended ask, and is dropped: %v", tellerevents.AskEnded, err)
return nil
}
if asksKV == nil {
return errors.New("the assigner's asks are not open")
}
var current []byte
var rev uint64
entry, err := asksKV.Get(ctx, e.Key)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
current, rev = entry.Value(), entry.Revision()
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(e.Key); err == nil && k.Kind == asks.AskKindRecord && putsKV != nil {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
pe, err := putsKV.Get(ctx, pkey)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
if p, err := puts.Decode(pe.Value()); err == nil {
put = &p
}
}
}
}
value, why := assign.AskedAgain(e, current, put, now)
if why == "" && stale != nil {
// An ask the pass takes away is not written again: it is about a move or a composition that has been left behind, and the
// pass removes it (issue 1966 note 165).
if s := stale(e.Key); s != "" {
why = "it is stale and left to the pass to take away: " + s
}
}
if why != "" {
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is not written again: %s", e.Key, e.Ask, e.Outcome, why)
return nil
}
if _, err := asksKV.Update(ctx, e.Key, value, rev); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
logf("%s: the ask %s ended (%s) without an answer, and the key changed since: it is not written again", e.Key, e.Ask, e.Outcome)
return nil
}
return err
}
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is written again under a new asking, which the teller asks", e.Key, e.Ask, e.Outcome)
return nil
}
@@ -0,0 +1,161 @@
package main
import (
"context"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-assigner/internal/assign"
)
// AssignerModule is the asker the assigner's compose asks name.
const AssignerModule = "mesh-delivery-assigner"
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
// second ask builds nothing.
type composeAsks struct {
ctx context.Context
asker buildrun.Asker
wake func()
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
asks map[string]*composeAsk
}
type composeAsk struct {
id string
cancel context.CancelFunc
ran *assign.Ran
}
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
}
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
// A job its layout refuses ends at once, failed with why: never silent.
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
a.mu.Lock()
defer a.mu.Unlock()
if k, ok := a.asks[key]; ok {
return k.ran, k.id
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
At: a.now(), Compose: &job}
ctx, cancel := context.WithCancel(a.ctx)
k := &composeAsk{id: ask.ID, cancel: cancel}
a.asks[key] = k
if err := a.asker.Put(ctx, ask); err != nil {
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
cancel()
return k.ran, k.id
}
go func() {
step, err := a.asker.Wait(ctx, ask)
if ctx.Err() != nil {
return
}
ran := assign.RanOf(step)
if err != nil {
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
}
a.mu.Lock()
k.ran = &ran
a.mu.Unlock()
a.wake()
}()
return nil, k.id
}
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
func (a *composeAsks) sweep(seen map[string]bool) {
a.mu.Lock()
defer a.mu.Unlock()
for key, k := range a.asks {
if !seen[key] {
k.cancel()
delete(a.asks, key)
}
}
}
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
// asks. The keys it used are marked in seen.
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
waiting := func(words string) delivery.CompositionVerdict {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
}
head, err := assign.HeadJob(m, ch)
if err != nil {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
}
var pending []string
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
seen[key] = true
headRan, id := p.asks.get(key, head)
if headRan == nil {
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
}
var readers []assign.Reader
var notJudged []string
for _, module := range sortedNames(assign.ReaderModules) {
repository := assign.ReaderModules[module]
var commits []string
member := false
if h := ch.Members[repository]; h != "" {
commits, member = []string{h}, true
} else {
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
}
if len(commits) == 0 {
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
"commit on record, and the group holds no member of "+repository)
continue
}
for _, commit := range commits {
job, err := assign.ReaderJob(ch, module, repository, commit)
if err != nil {
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
continue
}
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
seen[rkey] = true
ran, rid := p.asks.get(rkey, job)
if ran == nil {
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
continue
}
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
}
}
if len(pending) > 0 {
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
}
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
v.Warnings = append(v.Warnings, notJudged...)
return v
}
func sortedNames(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -1,74 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §717: the assigner's asks are the seat mesh-wants' state asks, so every key its asks package writes is one of
// that state's key forms, read back to the same parts, and each value is the seat's Ask.
func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
const hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
const fp, digest = "src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
must := func(k string, err error) string {
t.Helper()
if err != nil {
t.Fatal(err)
}
return k
}
three := func(parse func(string) (string, string, string, bool)) func(string) []string {
return func(k string) []string {
if a, b, c, ok := parse(k); ok {
return []string{a, b, c}
}
return nil
}
}
for _, row := range []struct {
key string
parse func(string) []string
want []string
}{
{must(asks.HoldAskKey("web", fp)), func(k string) []string {
if a, b, ok := meshwants.ParseAsksByHoldModuleFingerprintKey(k); ok {
return []string{a, b}
}
return nil
}, []string{"web", hex}},
{must(asks.RestoreAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRestoreNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.BackupAskKey("ace", "web", fp)), three(meshwants.ParseAsksByBackupNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.OutsideAskKey("ace", "web", fp)), three(meshwants.ParseAsksByOutsideNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RevertAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRevertNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RecordAskKey("ace", "web", digest)), three(meshwants.ParseAsksByRecordNodeModuleDigestKey), []string{"ace", "web", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}},
{must(asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")), func(k string) []string {
if a, ok := meshwants.ParseAsksByAssignmentsProposalKey(k); ok {
return []string{a}
}
return nil
}, []string{"p-20261028t183000-1a2b3c"}},
} {
got := row.parse(row.key)
if len(got) != len(row.want) {
t.Errorf("%s is no key form of mesh-wants' asks: read as %v", row.key, got)
continue
}
for i := range got {
if got[i] != row.want[i] {
t.Errorf("%s reads as %v, want %v", row.key, got, row.want)
break
}
}
}
key := must(asks.HoldAskKey("web", fp))
raw, err := asks.EncodeAsk(key, asks.Ask{Ask: json.RawMessage(`{"id":"x"}`), Asking: 2})
if err != nil {
t.Fatal(err)
}
if v, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
}
}
@@ -1,170 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the
// builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's
// `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard.
// builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded).
type builtEnds struct {
mu sync.Mutex
ends map[string]builds.BuildEnded
}
func (b *builtEnds) key(module, fp string) string { return module + "@" + fp }
// put takes one build.ended in.
func (b *builtEnds) put(data []byte) error {
var e builds.BuildEnded
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a build.ended that is no end: %w", err)
}
if e.Module == "" || e.Fingerprint == "" {
return fmt.Errorf("a build's end naming no module or fingerprint: %s", data)
}
if e.Outcome == "" { // an end said before A-162 is a done build's
e.Outcome = builds.BuildDone
}
b.mu.Lock()
defer b.mu.Unlock()
if b.ends == nil {
b.ends = map[string]builds.BuildEnded{}
}
b.ends[b.key(e.Module, e.Fingerprint)] = e
return nil
}
// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended.
func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) {
b.mu.Lock()
defer b.mu.Unlock()
e, ok := b.ends[b.key(module, fp)]
return e, ok
}
// partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `<module>:<artifact>`), with the
// node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose
// parts the version's release says (releasedVersions).
func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) {
b.mu.Lock()
e, ok := b.ends[b.key(module, fp)]
b.mu.Unlock()
if !ok || (e.KeptOn == "" && e.Branch == "") {
return assign.KeptBuild{}, false
}
parts := map[string]string{}
for name, digest := range e.Made {
parts[module+":"+name] = digest
}
return assign.KeptBuild{Parts: parts, KeptOn: e.KeptOn}, true
}
// releasedVersions are the catalog's version-released heard, by module and source revision.
type releasedVersions struct {
mu sync.Mutex
seen map[string]assign.KeptBuild
// newest is, by module, the last version-released heard of it (delivered.go).
newest map[string]releaseHeard
}
func (r *releasedVersions) key(module string, source int64) string {
return fmt.Sprintf("%s@%d", module, source)
}
// put takes one version-released in.
func (r *releasedVersions) put(data []byte) error {
var e meshcatalog.VersionReleasedEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-released that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-released naming no module or version: %s", data)
}
parts := map[string]string{}
for _, a := range e.Artifacts {
parts[e.Module+":"+a.Artifact] = a.Digest
}
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]assign.KeptBuild{}
}
r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts}
if r.newest == nil {
r.newest = map[string]releaseHeard{}
}
h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source}
if e.At != nil {
h.At = *e.At
}
r.newest[e.Module] = h // the stream's order is the order of release
return nil
}
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
r.mu.Lock()
defer r.mu.Unlock()
k, ok := r.seen[r.key(module, source)]
return k, ok
}
// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and
// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703).
var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream)
// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was
// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before
// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner
// starts again.
func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) {
if err := ends.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf)
if err != nil {
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
return
}
nh.Heard(buildsStreamName)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearTaken registers the hub's taker of the indexer's take-in.taken, each waking the pass (orderwaits.go, §298, A-260); before
// the hub starts.
func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf func(string, ...any)) error {
return hub.Handle(takenSubject, func(m jetstream.Msg) {
if err := taken.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
})
}
@@ -1,110 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
)
// Design 47 G §379, §526, A-96: a build.ended heard names a kept build's or another branch's parts by digest (`<module>:<artifact>`) and its
// keeping node; a build of the trunk not kept names none, its parts being the release's.
func TestABuildEndedHeardNamesAKeptBuildsPartsAndItsNode(t *testing.T) {
put := func(b *builtEnds, e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := b.put(raw); err != nil {
t.Fatal(err)
}
}
made := map[string]string{"program": "sha256:1"}
var b builtEnds
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "a", Made: made})
if _, ok := b.partsOf("node-engine", "a"); ok {
t.Error("a trunk build not kept names parts")
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "b", Made: made, KeptOn: "ace"})
if k, ok := b.partsOf("node-engine", "b"); !ok || k.KeptOn != "ace" || !reflect.DeepEqual(k.Parts, map[string]string{"node-engine:program": "sha256:1"}) {
t.Errorf("kept build = %+v, %v", k, ok)
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "c", Made: made, Branch: "cand"})
if k, ok := b.partsOf("node-engine", "c"); !ok || k.KeptOn != "" || len(k.Parts) != 1 {
t.Errorf("branch build = %+v, %v", k, ok)
}
if err := b.put([]byte(`{"module":"x"}`)); err == nil {
t.Error("an end naming no fingerprint is kept")
}
}
// Design 47 G §379, A-96: a version-released heard names the version and the parts' digests of the module's source revision.
func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
var r releasedVersions
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := r.put(raw); err != nil {
t.Fatal(err)
}
want := assign.KeptBuild{Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}}
if k, ok := r.of("mail", 7); !ok || !reflect.DeepEqual(k, want) {
t.Errorf("released = %+v, %v", k, ok)
}
if _, ok := r.of("mail", 8); ok {
t.Error("another source revision names a version")
}
}
// Design 47 G §379, A-96: the trunk's entry takes its version and parts from the version-released of the kept assignment graph
// revision's source; a kept build from its end; no revision kept, or another fingerprint, names none.
func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *testing.T) {
w := newWorld()
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := w.released.put(raw); err != nil {
t.Fatal(err)
}
endRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "k", Made: map[string]string{"image": "sha256:2"}, KeptOn: "g14"})
if err := w.ends.put(endRaw); err != nil {
t.Fatal(err)
}
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
named := p.namedBuild("mail", "t")
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" || k.KeptOn != "" {
t.Errorf("trunk entry = %+v, %v", k, ok)
}
if k, ok := named("k"); !ok || k.KeptOn != "g14" || k.Version != "" {
t.Errorf("kept build = %+v, %v", k, ok)
}
if _, ok := named("other"); ok {
t.Error("another fingerprint names a release")
}
p.graph.source = 8
if _, ok := named("t"); ok {
t.Error("a source with no release names a version")
}
p.graph.ok = false
if _, ok := p.namedBuild("mail", "t")("t"); ok {
t.Error("no revision kept names a release")
}
}
// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed.
func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) {
w := newWorld()
say := func(e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := w.ends.put(raw); err != nil {
t.Fatal(err)
}
}
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a build nothing was heard of failed")
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"})
if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" {
t.Errorf("failed end: failed %v why %q", failed, why)
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone})
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a later done end leaves the build failed")
}
}
@@ -1,24 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G
// A-181: what was built is heard, never read from the builder's bucket).
func endDone(t testing.TB, w *world, module, fp, manifest string) {
t.Helper()
data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest})
if err != nil {
t.Fatal(err)
}
if err := w.ends.put(data); err != nil {
t.Fatal(err)
}
}
// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507).
const trunkBranch = "main"
@@ -1,50 +0,0 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// addedOf is the `added` of node's candidate (design 47 G A-316, §379, §421): by module, each module the candidate adds to node
// (laid over current) with no want there, the version and parts the want it would write carries, taken from the same code that
// writes a want's (namedBuild, the want's build at the fingerprint of the branch the want's code takes). A module whose build names none is not listed.
func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]mma.Assignments) map[string]mma.AddedModule {
if laid == nil {
return nil
}
placed := map[string]string{}
for _, m := range laid.Modules {
placed[m.Module] = m.Branch
}
var out map[string]mma.AddedModule
for _, module := range plc.ChangeOf(current[node], *laid).Added {
if key, err := mwc.WantKey(node, module); err != nil || p.w.has(buckets.Wants(), key) {
continue
}
// The branch the want's code takes: the one the candidate's placement names, else the module's default (groups, wants).
branch := p.w.trunkName()
if p.ends.following() {
branch = p.ends.branchOf(module, placed[module])
}
fp := p.w.trunkOf(branch)[module].Fp2
if fp == "" {
continue
}
k, ok := p.namedBuild(module, fp)(fp)
if !ok {
continue
}
if out == nil {
out = map[string]mma.AddedModule{}
}
out[module] = mma.AddedModule{Version: k.Version, Parts: k.Parts}
}
return out
}
// candidateOf is the candidate of proposal id for node: the assignments laid over it, and what it adds (addedOf).
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
}
@@ -1,59 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"testing"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G A-316: the candidate of a node carries, for each module it adds there without a want, the version and parts the
// want it would write carries; a module with a want there, or a build that names none, is not listed.
func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
w := newWorld()
for _, m := range []string{"mail", "web", "db"} {
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
}
w.first = "main"
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := w.released.put(rel); err != nil {
t.Fatal(err)
}
end, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: "fp-db", Made: map[string]string{"program": "sha256:3"}, KeptOn: "g14"})
if err := w.ends.put(end); err != nil {
t.Fatal(err)
}
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
c := p.candidateOf("p1", "ace", laid, current)
want := map[string]mma.AddedModule{
"mail": {Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}},
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
}
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
t.Errorf("candidate = %+v, added want %+v", c, want)
}
if got := p.candidateOf("p1", "ace", nil, current); got.Added != nil {
t.Errorf("a candidate taking the key away adds %+v", got.Added)
}
// A placement on a watched branch is added at that branch's fingerprint, as the want's code takes it, not the trunk's.
w.putModule("feat", "mail", meshcatalog.ModulesByModuleValue{Fp2: "fp-feat"}, graph.Node{}, nil)
feat, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-feat", Made: map[string]string{"program": "sha256:4"}, KeptOn: "g14"})
if err := w.ends.put(feat); err != nil {
t.Fatal(err)
}
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = w.trunkName
p.ends.on.Store(true)
onFeat := &mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "mail", Branch: "feat"}}}
if got, want := p.candidateOf("p2", "g14", onFeat, nil).Added, map[string]mma.AddedModule{"mail": {Parts: map[string]string{"mail:program": "sha256:4"}}}; !reflect.DeepEqual(got, want) {
t.Errorf("a placement on branch feat adds %+v, want %+v", got, want)
}
}
@@ -1,113 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"strings"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/assigngraph"
)
// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-<id>`'s catalog graph revision,
// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is
// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate.<group>.<heads>;
// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing.
// runGroupPrefix is a run on request's candidate group before its id, `run-<id>` (A-160).
const runGroupPrefix = "run-"
// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the
// candidate's group and heads and the revision its event names.
func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
after := ""
for {
a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after})
if err != nil {
return half, err
}
if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group {
return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision)
}
half.Revision = *a.Revision
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
after = a.Next
}
}
// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a
// group's) is passed over.
func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error {
if !strings.HasPrefix(ev.Group, runGroupPrefix) {
return nil
}
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return errors.New("the assigner keeps no assignment graph, so no candidate's is kept")
}
half, err := readCandidateHalf(call, ev)
if err != nil {
return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err)
}
in := p.graphInputs()
in.Halves = maps.Clone(in.Halves)
in.Halves[half.Revision.Branch] = half
c := assigngraph.Compute(in)
for _, u := range c.Uncomputed {
p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u)
}
rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock())
if err != nil {
return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err)
}
if fresh {
p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision)
}
return nil
}
// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235).
func (p *passer) dropCandidate(ctx context.Context, group string) error {
g := p.graph
if g == nil || g.objects == nil {
return nil
}
o, ok := g.objects.(assigngraph.Remover)
if !ok {
return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group)
}
n, err := assigngraph.DropCandidate(ctx, o, group)
if err == nil && n > 0 {
p.logf("the candidate %s's assignment graph is dropped: its run ended", group)
}
return err
}
// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it.
func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
raw, err := call(mcc.RevisionVerb, in)
if err != nil {
return meshcatalog.RevisionAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
return a, nil
}
@@ -0,0 +1,180 @@
//go:build integration
package main
import (
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
type holder struct {
mu sync.Mutex
asks []delivery.StepAsk
}
func (h *holder) taken() []delivery.StepAsk {
h.mu.Lock()
defer h.mu.Unlock()
return append([]delivery.StepAsk(nil), h.asks...)
}
func (b *bed) hold(t *testing.T) *holder {
t.Helper()
seat, _ := delivery.AskStream(delivery.AskGo)
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
h := &holder{}
cc, err := cons.Consume(func(m jetstream.Msg) {
a, err := delivery.DecodeStepAsk(m.Data())
if err != nil || a.Compose == nil {
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
m.Ack()
return
}
h.mu.Lock()
h.asks = append(h.asks, a)
h.mu.Unlock()
req, err := composer.Decode(a.Compose.Request)
if err == nil {
err = composer.Fill(&req, func(digest string) ([]byte, error) {
key, _ := delivery.ManifestsKey(digest)
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
if err != nil {
return nil, err
}
man, err := delivery.DecodeManifest(e.Value())
return []byte(man.JSON), err
})
}
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else if v, err := composer.Run(req); err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else {
vr, _ := json.Marshal(v)
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
}
raw, err := delivery.EncodeStep(step)
if err != nil {
t.Errorf("the step cannot be written: %v", err)
}
key, _ := delivery.AskStepKey("ace", a.ID)
if _, err := steps.Put(b.ctx, key, raw); err != nil {
t.Errorf("the step %s: %v", key, err)
}
m.Ack()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(cc.Stop)
return h
}
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// head's composer was built with, and asks nothing more on a later pass of the same heads.
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
running := strings.Repeat("3", 40)
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey("feat/x")
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
b.start(t)
key, _ := delivery.CompositionKey("feat/x", hash)
// No holder takes the asks yet: the composition waits, naming them.
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
}
h := b.hold(t)
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
t.Fatalf("composition = %+v, want a pass on g14", v)
}
asks := h.taken()
if len(asks) != 2 {
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
}
byRepo := map[string]delivery.StepAsk{}
for _, a := range asks {
byRepo[a.Compose.Fetch["repository"].(string)] = a
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
t.Errorf("ask = %+v", a)
}
}
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
t.Errorf("the head's ask = %+v", a.Compose)
}
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running controller's ask = %+v", a.Compose)
}
// A later pass of the same heads (woken by any change) asks nothing more.
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
time.Sleep(500 * time.Millisecond)
if n := len(h.taken()); n != 2 {
t.Errorf("a later pass of the same heads asked again: %d asks", n)
}
}
// verdict waits for the composition under key to satisfy ok, and answers it.
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
t.Helper()
var last delivery.CompositionVerdict
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
t.Fatal(err)
}
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
t.Fatal(err)
}
if ok(last) {
return last
}
}
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
return last
}
@@ -1,78 +0,0 @@
package main
import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
// it was refused or cannot be read.
type publishedRow struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp,omitempty"`
Digest string `json:"digest,omitempty"`
Refused string `json:"refused,omitempty"`
Unread string `json:"unread,omitempty"`
}
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
// names its node first): each value decoded and digested here, as the node-engine digests it (nodeengineclient.ComposedDigest).
// A value or key that does not read is named with why, never left out.
func published(values map[string][]byte) map[string][]publishedRow {
byNode := map[string]map[string][]byte{}
for key, raw := range values {
node, _, _ := strings.Cut(key, ".")
if byNode[node] == nil {
byNode[node] = map[string][]byte{}
}
byNode[node][key] = raw
}
out := map[string][]publishedRow{}
for node, keys := range byNode {
var rows []publishedRow
for key, raw := range keys {
row := publishedRow{Key: key}
_, module, part, isNode, err := nodeengineclient.ParseComposedKey(key)
var fp string
if err == nil && !isNode {
fp, err = catalogkey.ParseKeyPart(part)
}
switch {
case err != nil:
row.Unread = err.Error()
case isNode:
row.Module = "node"
default:
row.Module, row.FP = module, fp
}
if err == nil {
c, derr := nodeengineclient.DecodeComposition(raw)
switch {
case derr != nil:
row.Unread = derr.Error()
case c.Refused != "":
row.Refused = c.Refused
default:
if d, err := nodeengineclient.ComposedDigest(c); err != nil {
row.Unread = err.Error()
} else {
row.Digest = d
}
}
}
rows = append(rows, row)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
out[node] = rows
}
return out
}
@@ -1,165 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"sync"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
// asked of the machine's node-engine by the seat's verb compose-candidate.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
// candidateKeyOf is the key the step's pipeline names its candidate by (A-226): a run on request's `run-<id>`, or for a branch
// pipeline the branch's catalog graph revision `<branch>.<revision>`, the branch escaped as a key part as the pipeline key does.
func candidateKeyOf(run meshwants.StepRun) (string, error) {
switch {
case run.Run != "":
return "run-" + run.Run, nil
case run.Branch == "" || run.Revision <= 0:
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
}
return keys.EscapeName(run.Branch) + "." + strconv.FormatInt(run.Revision, 10), nil
}
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
// naming none is the trunk's, trunk the first watched branch); a branch that is not watched takes the nodes placing the module on the trunk. In name order.
func machinesFor(placed map[string]map[string]string, module, branch, trunk string, watched bool) []string {
if branch == "" || !watched {
branch = trunk
}
var out []string
for node, b := range placed[module] {
if b == "" {
b = trunk
}
if b == branch {
out = append(out, node)
}
}
slices.Sort(out)
return out
}
// machinesOf are the machines of a composition step in the world's assignments.
func (w *world) machinesOf(module, branch string) []string {
return machinesFor(w.placedBranches(), module, branch, w.trunkName(), w.holdsBranch(branch))
}
// nodeInput is one machine's input: its bytes, or why the node gave none, or that it is away.
type nodeInput struct {
Node string
Body []byte
// Why is set when the node gave no input: its words, or the call's error.
Why string
// Away is a machine the watchman said away: it is not asked (§64).
Away bool
}
// compositionInputs asks the machines of a step for their inputs.
type compositionInputs struct {
machines func(module, branch string) []string
// call asks one node's seat verb (process.Conn.CallKeptOn).
call func(node, key string, body any) (json.RawMessage, error)
bound time.Duration
}
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
}
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() {
defer wg.Done()
out[i] = c.one(ctx, bound, node, key)
}()
}
wg.Wait()
return out, nil
}
// one asks one node for its input.
func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput {
in := nodeInput{Node: node}
at, cancel := context.WithTimeout(ctx, bound)
defer cancel()
type reply struct {
raw json.RawMessage
err error
}
got := make(chan reply, 1)
go func() {
raw, err := c.call(node, composeCandidateKey, ne.ComposeCandidateInput{Candidate: key, Input: true})
got <- reply{raw, err}
}()
var raw json.RawMessage
select {
case <-at.Done():
in.Why = "no answer within its bound: " + at.Err().Error()
return in
case r := <-got:
if r.err != nil {
in.Why = r.err.Error()
return in
}
raw = r.raw
}
var a ne.ComposeCandidateAnswer
// its numbers as written (json.Number), so the input kept is the engine's own bytes' values
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
if err := dec.Decode(&a); err != nil {
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
return in
}
switch {
case a.Verdict == nec.Unready:
in.Why = "unready: " + a.Refused
return in
case len(a.Input) == 0:
in.Why = "the engine answered no composition input"
if a.Refused != "" {
in.Why += ": " + a.Refused
}
return in
}
body, err := json.Marshal(a.Input)
if err != nil {
in.Why = "its composition input cannot be encoded again: " + err.Error()
return in
}
in.Body = body
return in
}
@@ -1,96 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G A-226: the candidate a step asks is `run-<id>` for a run on request, else `<branch>.<revision>`.
func TestTheCandidateKeyOfAStep(t *testing.T) {
for _, c := range []struct {
run meshwants.StepRun
want string
fail bool
}{
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
{run: meshwants.StepRun{Branch: "main"}, fail: true},
} {
got, err := candidateKeyOf(c.run)
if (err != nil) != c.fail || got != c.want {
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
}
}
}
// A-252: the machines of a step are the nodes whose placement names its branch; none named is the trunk; an unwatched branch takes
// the trunk's nodes.
func TestTheMachinesOfAStep(t *testing.T) {
placed := map[string]map[string]string{"m": {"ace": "", "g14": "feat", "novox": "main", "shanks": "feat"}, "other": {"ace": ""}}
for _, c := range []struct {
branch string
watched bool
want string
}{
{"main", true, "ace novox"},
{"feat", true, "g14 shanks"},
{"gone", false, "ace novox"},
{"", true, "ace novox"},
} {
if got := strings.Join(machinesFor(placed, "m", c.branch, trunkBranch, c.watched), " "); got != c.want {
t.Errorf("branch %q watched %v: %q, want %q", c.branch, c.watched, got, c.want)
}
}
if got := machinesFor(placed, "none", "main", trunkBranch, true); len(got) != 0 {
t.Errorf("a module placed nowhere has machines %v", got)
}
}
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// gather asks the machines side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(node, key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
mu.Lock()
asked[node] = key + " " + string(raw)
mu.Unlock()
switch node {
case "g14":
return nil, errors.New("no responders")
case "novox":
return json.RawMessage(`{"candidate":"run-r1","node":"novox","verdict":"unready","refused":"starting","modules":[],"own":{},"took":1,"v":1}`), nil
}
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, call: call, bound: time.Second}
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
if want := `seat:node-engine.compose-candidate {"candidate":"run-r1","input":true}`; asked["ace"] != want {
t.Errorf("asked %q, want %q", asked["ace"], want)
}
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
t.Errorf("ace: %+v", got[0])
}
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
}
if _, was := asked["shanks"]; was || !got[3].Away {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, call: call}
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
@@ -1,506 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
// (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
// asks are put on and the first of their step keys (§467, §481, §600, A-163).
runAsker = meshwants.Seat
// changeCase is the one case each request composes.
changeCase = "change"
// inputVerbKey is the verb a run ask's input is served on: the seat mesh-wants' input (§354).
inputVerbKey = "seat:" + meshwants.Seat + "." + meshwants.VerbInput
// entryVerbKey and seatDefinitionsKey are the mesh-catalog seat verbs the step reads.
entryVerbKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
type runEnds struct {
mu sync.Mutex
waits map[string]chan buildstep.StepEnded
}
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
r.mu.Lock()
defer r.mu.Unlock()
if r.waits == nil {
r.waits = map[string]chan buildstep.StepEnded{}
}
ch := make(chan buildstep.StepEnded, 1)
r.waits[id] = ch
return ch
}
// forget stops waiting for the ask id.
func (r *runEnds) forget(id string) {
r.mu.Lock()
delete(r.waits, id)
r.mu.Unlock()
}
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
func (r *runEnds) heard(e buildstep.StepEnded) {
r.mu.Lock()
ch := r.waits[e.Ask]
delete(r.waits, e.Ask)
r.mu.Unlock()
if ch != nil {
ch <- e
}
}
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
type awayNodes struct {
mu sync.Mutex
away map[string]bool
}
// set keeps machine away (true) or back (false).
func (a *awayNodes) set(machine string, away bool) {
a.mu.Lock()
defer a.mu.Unlock()
if a.away == nil {
a.away = map[string]bool{}
}
if away {
a.away[machine] = true
} else {
delete(a.away, machine)
}
}
// is says machine is away.
func (a *awayNodes) is(machine string) bool {
if a == nil {
return false
}
a.mu.Lock()
defer a.mu.Unlock()
return a.away[machine]
}
// compositionRun runs a composition step.
type compositionRun struct {
inputs *compositionInputs
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
call func(key string, body any) (json.RawMessage, error)
// trunk is the trunk's catalog half as the assigner holds it.
trunk func() (mcc.Half, bool)
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
running func(node, module string) (ne.Running, bool)
away *awayNodes
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
logf func(string, ...any)
}
// machineEnd is what one machine came to.
type machineEnd struct {
failed, couldNot, warning string
}
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed, else passed with a warning per machine away or not judged.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(half.Revision.Definitions)
if err != nil {
return couldNotRun(err.Error())
}
manifests, verdicts, err := caseOf(half)
if err != nil {
return couldNotRun(err.Error())
}
programOf, err := r.programs(half)
if err != nil {
return couldNotRun(err.Error())
}
inputs, err := r.inputs.gather(ctx, run, r.away.is)
if err != nil {
return couldNotRun(err.Error())
}
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
ends := make([]machineEnd, len(inputs))
var wg sync.WaitGroup
for i, in := range inputs {
wg.Add(1)
go func() {
defer wg.Done()
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
}()
}
wg.Wait()
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
var out stepOutcome
var failed, couldNot []string
for _, e := range ends {
failed = appendSome(failed, e.failed)
couldNot = appendSome(couldNot, e.couldNot)
out.Warnings = appendSome(out.Warnings, e.warning)
}
switch {
case len(failed) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
default:
out.Outcome = meshwants.StepEndedOutcomePassed
}
return out
}
func appendSome(to []string, s string) []string {
if s == "" {
return to
}
return append(to, s)
}
func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
node := in.Node
switch {
case in.Away:
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
case in.Why != "":
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
}
program, err := programOf(node)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
raw, err := json.Marshal(req)
if err != nil {
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
}
if _, err := r.keeper.keep(ask, raw); err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
ended := r.ends.expect(ask)
defer r.ends.forget(ask)
if err := r.put(ctx, a); err != nil {
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
}
var e buildstep.StepEnded
select {
case <-ctx.Done():
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
case e = <-ended:
}
if e.Outcome != buildstep.OutcomePassed {
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
}
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
if err != nil {
return machineEnd{couldNot: node + ": " + err.Error()}
}
return verdictOf(node, answer)
}
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged passes with a warning.
func verdictOf(node string, answer []byte) machineEnd {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
}
if v.V != composer.Version {
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
switch {
case !ok:
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
case m.NotJudged != "":
return machineEnd{warning: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
}
return machineEnd{}
}
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
raw, err := r.call(mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(body, &a); err != nil {
return mcc.Half{}, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != run.Revision {
return mcc.Half{}, fmt.Errorf("%s answered no revision %d of %q", mcc.RevisionVerb, run.Revision, run.Branch)
}
if in.After == "" {
half.Revision = *a.Revision // the first page names what the revision was read from, its definitions included
}
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
in.After = a.Next
}
}
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
// neither holds is an error naming the seat and the digest.
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
}
def, err := json.Marshal(a.Entry.Definition)
if err != nil {
return nil, err
}
out = append(out, def)
}
return out, nil
}
// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not
// accepted (a module without one is accepted).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) {
manifests := map[string]composer.Manifest{}
verdicts := map[string]json.RawMessage{}
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
manifests[name] = composer.Manifest{Raw: raw}
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
continue
}
v := meshcatalog.VerdictsByDigestValue{V: 1, Module: name, State: e.Verdict}
if e.Verdict != mcc.VerdictRefused {
v.State = mcc.VerdictManifestWaits
}
if e.Inputs != nil {
v.Problems, v.Unknown, v.Warnings = e.Inputs.Problems, e.Inputs.Unknown, e.Inputs.Warnings
}
raw, err = json.Marshal(v)
if err != nil {
return nil, nil, err
}
verdicts[name] = raw
}
return manifests, verdicts, nil
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(mapsKeys(entries)) {
for _, s := range entries[name].Serves {
if strings.HasPrefix(s, ne.Seat+".") {
return entries[name], true
}
}
}
return meshcatalog.CatalogEntry{}, false
}
func mapsKeys(m map[string]meshcatalog.CatalogEntry) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
// programOfEntry is the composer built from a node-engine entry: its repository, commit and directory, the composer's package.
func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
dir := e.Path
if dir == "" {
dir = "."
}
if e.Repository == "" || e.Commit == "" {
return buildstep.RunProgram{}, fmt.Errorf("the node-engine's entry %s names no repository or commit", e.Fp)
}
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`.
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
}
var mu sync.Mutex
byFp := map[string]buildstep.RunProgram{}
return func(node string) (buildstep.RunProgram, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
return buildstep.RunProgram{}, fmt.Errorf("no build of %s is heard running there", engine.Module)
}
mu.Lock()
p, ok := byFp[running.Fingerprint]
mu.Unlock()
if ok {
return p, nil
}
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
return p, err
}
mu.Lock()
byFp[running.Fingerprint] = p
mu.Unlock()
return p, nil
}, nil
}
// heardPresence keeps a machine-away (away true) or machine-back of the seat mesh-watchdogs (§389, §436): the machine it names.
func heardPresence(a *awayNodes, away bool, m jetstream.Msg, logf func(string, ...any)) {
var e struct {
Machine string `json:"machine"`
}
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Machine == "" {
logf("%s holds no machine, left: %v", m.Subject(), err)
} else {
a.set(e.Machine, away)
if away {
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", e.Machine)
} else {
logf("%s is back (machine-back): a composition step composes it again", e.Machine)
}
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
}
// trunkOf is the trunk's catalog half the assigner's views hold, trunk naming the first watched branch, or false while none is
// held.
func trunkOf(iv *catalogViews, trunk func() string) func() (mcc.Half, bool) {
return func() (mcc.Half, bool) {
if iv == nil || iv.h == nil {
return mcc.Half{}, false
}
h, ok := iv.h.views()[trunk()]
return h, ok
}
}
// runAskPublisher puts a run ask on the Go build seat's check queue under the assigner's name (`accept.check.assigner`, §600), its
// id the message id, so the stream takes it once.
func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildstep.StepAsk) error {
return func(ctx context.Context, a buildstep.StepAsk) error {
raw, err := buildstep.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, a.Asker)
if err != nil {
return err
}
_, err = js.Publish(ctx, subject, raw, jetstream.WithMsgID(a.ID))
return err
}
}
@@ -1,176 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
)
const (
commitE1 = "1111111111111111111111111111111111111111"
commitE2 = "2222222222222222222222222222222222222222"
commitE3 = "3333333333333333333333333333333333333333"
)
// runStand is a composition run over stand-ins of the mesh-catalog seat, the machines' engines, the build seat and its step-ended.
type runStand struct {
mu sync.Mutex
asks map[string]buildstep.StepAsk // by node
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
}
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
engine := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
Manifest: map[string]any{"module": "engine"}}
}
revisionCommit := commitE1
if revisionEngine != "src2:e1" {
revisionCommit = commitE3
}
pages := map[string]meshcatalog.RevisionAnswer{
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
switch key {
case mcc.RevisionVerb:
var in meshcatalog.RevisionInput
_ = json.Unmarshal(raw, &in)
if in.Revision != 7 || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
return json.Marshal(pages[in.After])
case seatDefinitionsKey:
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
case entryVerbKey:
var in meshcatalog.EntryInput
_ = json.Unmarshal(raw, &in)
s.mu.Lock()
s.entries = append(s.entries, in.Fingerprint)
s.mu.Unlock()
commit := map[string]string{"src2:e1": commitE1, "src2:e2": commitE2}[in.Fingerprint]
return json.Marshal(meshcatalog.EntryAnswer{Entry: engine(in.Fingerprint, commit)})
}
return nil, errors.New("no such verb " + key)
}
engines := func(node, _ string, _ any) (json.RawMessage, error) {
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
away := &awayNodes{}
away.set("shanks", true)
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
ends := &runEnds{}
s.r = &compositionRun{
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, call: engines, bound: time.Second},
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
running: func(node, module string) (ne.Running, bool) {
fp, ok := running[node]
return ne.Running{Node: node, Module: module, Fingerprint: fp}, ok && module == "engine"
},
put: func(_ context.Context, a buildstep.StepAsk) error {
body, err := keeper.body(a.Run.Input.Digest)
if err != nil {
return err
}
req, err := composer.Decode(body)
if err != nil {
return err
}
node := req.Machines[0]
s.mu.Lock()
s.asks[node], s.requests[node] = a, req
s.mu.Unlock()
m := composer.Machine{Composes: true}
if node == "g14" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
return nil
},
}
return s
}
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
// depth compose with the one machine and one case `change`, its manifests, verdicts and seats from the step's revision; the
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
// trunk's; the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
out := s.r.run(context.Background(), "ask-step", step)
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
t.Fatalf("the step came to %+v", out)
}
if len(s.asks) != 2 {
t.Fatalf("asked %d runs, want one per machine not away", len(s.asks))
}
a, req := s.asks["ace"], s.requests["ace"]
if a.Kind != buildstep.AskCheck || a.Language != buildstep.AskGo || a.Asker != runAsker || a.Run == nil || len(a.Run.Heads) != 0 ||
a.Run.Input.Verb != "seat:mesh-wants.input" || a.Run.Program != (buildstep.RunProgram{Repository: "novox/mesh-host", Commit: commitE1, Dir: ".", Package: composer.Package}) {
t.Errorf("the run ask of ace %+v %+v", a, a.Run)
}
if s.asks["g14"].Run.Program.Commit != commitE2 {
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
}
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
t.Errorf("the request of ace %+v", req)
}
c := req.Cases[0]
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
}
if strings.Join(s.entries, " ") == "" {
t.Error("no node-engine entry is read through entry")
}
// The revision's node-engine differs from the trunk's: that build composes every machine, and no entry is asked.
changed := newRunStand(t, "src2:e3", "src2:e1", "d1")
if out := changed.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed {
t.Fatalf("the step came to %+v", out)
}
if changed.asks["ace"].Run.Program.Commit != commitE3 || changed.asks["g14"].Run.Program.Commit != commitE3 || len(changed.entries) != 0 {
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
}
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "s1") || !strings.Contains(out.Said, "d1") || len(other.asks) != 0 {
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
}
// At the inputs state's cap the ask is refused, naming the store (§354).
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
if out := full.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "the state inputs is at its cap") || len(full.asks) != 0 {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
@@ -1,168 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
// composition step of one module's pipeline that the runner asks, runs it, and says its end as the seat's event
// step-ended.<node>, the ask's id as the event's id and the event's `id`, which the runner holds the end to.
// stepOutcome is what one composition step came to.
type stepOutcome struct {
Outcome string
Said string
Warnings []string
}
// compositionSteps takes the composition steps and says their ends.
type compositionSteps struct {
node string
// say says one event of the seat mesh-wants (wantsPublisher).
say publisher
// run runs one step and answers what it came to.
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
now func() time.Time
logf func(string, ...any)
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
// accepted kind step, read from the seat list when it is said, since that subject names no asker (design 47 G A-294, which
// corrects A-267). Empty while the seat list holds no definition naming exactly one.
func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
return func() string {
d, ok := definition()
if !ok {
return ""
}
var one string
for _, m := range d.Accepts {
if m.Name != meshwants.AcceptStep || m.Callers == nil {
continue
}
for _, e := range m.Callers.List {
name, isSeat := strings.CutPrefix(e, seatdef.CallerSeatPrefix)
if !isSeat || (one != "" && one != name) {
return ""
}
one = name
}
}
return one
}
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
type stepPhase struct {
mu sync.Mutex
phase string
}
type phaseKey struct{}
// setPhase sets the phase of the step ctx runs, when it runs one.
func setPhase(ctx context.Context, phase string) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.phase = phase
p.mu.Unlock()
}
}
func (p *stepPhase) get() string {
p.mu.Lock()
defer p.mu.Unlock()
return p.phase
}
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition step, or an ask with no
// id, is left with a log line; one taken is run in its own goroutine and its end said.
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
var run meshwants.StepRun
if err := json.Unmarshal(data, &run); err != nil || id == "" || run.Step != meshwants.StepRunStepComposition {
c.logf("a step ask that is no composition step with an id is left: %v", err)
return false
}
go c.runOne(ctx, id, run)
return true
}
// runOne runs one step and says its end.
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
started := c.now().UTC()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
ended := c.now().UTC()
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
}
}
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()) {
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(progressEvery)
done := make(chan struct{})
ended := make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
}
}
}()
return func() { close(done); <-ended }
}
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
func stepAskID(m *nats.Msg) string {
if id := m.Header.Get("x-event-id"); id != "" {
return id
}
return m.Header.Get(nats.MsgIdHdr)
}
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
@@ -1,224 +0,0 @@
//go:build integration
package main
import (
"bytes"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §354, §22, §63, §64, §389, A-240, A-252, A-254: the assigner started whole takes the runner's composition step of a
// module placed on three machines, one of them away: it asks the two others' node-engines for their inputs, puts one run ask per
// machine on the Go build seat's check queue under its name, its program the node-engine build each machine runs; the stand-in
// build seat reads each request by the seat mesh-wants' verb input in chunks and says its step-ended with the composer's verdict;
// the step's step-ended fails naming the machine that does not compose, warns of the machine away, and no input stays kept.
func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) {
b := startBed(t, "")
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const engineCommit = "1111111111111111111111111111111111111111"
// The assigner's durable on EVENTS hears, beside the bed's, the build seat's step-ended and the watchman's machine-away and
// machine-back, as the grants part derives them from its consumes; what each node runs it hears on its kept stream (the bed's).
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, meshwatchdogs.SubjectMachineAwayAll, meshwatchdogs.SubjectMachineBackAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
away, _ := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: "n3", At: time.Now().UTC(), LastHeard: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, meshwatchdogs.SubjectMachineAway("n3"), away); err != nil {
t.Fatal(err)
}
// The trunk's catalog half holds the module and the node-engine, the module claiming the seat node-engine.
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: engineCommit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine,
"m": {Module: "m", Fp: "src2:m1", Repository: "novox/m", Commit: engineCommit, Path: ".", Manifest: map[string]any{"module": "m", "version": "0.1"}}}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2", "n3"} {
b.place(t, node, "m")
}
// The node-engines of n1 and n2 answer compose-candidate with their inputs; n3 is never asked.
asked := make(chan string, 8)
engines, err := nc.Subscribe("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) {
node := strings.Split(m.Subject, ".")[5]
asked <- node
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"candidate": "main.3", "node": node, "verdict": "composed", "input": map[string]any{"node": node},
"modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}, "node": node})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engines.Unsubscribe() })
// mesh-catalog's entry answers the node-engine build's entry by its fingerprint.
entries, err := nc.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: engine}, "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = entries.Unsubscribe() })
subscribed(t, nc)
// The runner hears the step's end.
ended := make(chan meshwants.StepEnded, 1)
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data, &e)
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
select {
case <-b.seatListRead: // the seat server admits a caller of input once the seat list is read (A-278)
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", "step-ask-1")
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
// its end with the composer's verdict: n1 composes, n2 does not.
queue, _ := buildstep.AskStream(buildstep.AskGo)
// A-163: the assigner asks under its seat's own name.
subject := "mesh.seat.node-build-go.accept.check.mesh-wants"
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
programs := map[string]string{}
for range 2 {
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
if err != nil {
t.Fatalf("no run ask is put: %v", err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
node := req.Machines[0]
programs[node] = a.Run.Program.Commit
m := composer.Machine{Composes: true}
if node == "n2" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(v), At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
t.Fatal(err)
}
}
var e meshwants.StepEnded
select {
case e = <-ended:
case <-b.ctx.Done():
t.Fatal("the composition step said no end")
}
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
t.Fatalf("the step ended %+v", e)
}
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
t.Errorf("the composers were built at %v, want the node-engine build each machine runs", programs)
}
close(asked)
for node := range asked {
if node == "n3" {
t.Error("the machine away was asked for its input")
}
}
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
if len(keys) != 0 {
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
}
}
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
// chunks on the call's subjects, each acknowledged.
func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
t.Helper()
call := "standin" + strings.Repeat("0", 16)
chunks := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: call}
sub, err := nc.SubscribeSync(chunks.Chunks())
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
body, _ := json.Marshal(meshwants.InputInput{Digest: in.Digest, Call: call})
// On the call's subject naming its caller, the stand-in build seat's account: the seat server admits no unproven caller (A-179).
reply, err := nc.Request("mesh.seat.mesh-wants.call.input.n1~mesh-build-go", body, 10*time.Second)
if err != nil {
t.Fatal(err)
}
raw, said, ok := buscalls.Unwrap(reply.Data)
var a meshwants.InputAnswer
if !ok || json.Unmarshal(raw, &a) != nil || a.Chunks < 1 {
t.Fatalf("the verb input answered %s: %s", reply.Data, said)
}
var out bytes.Buffer
next := func() (string, []byte, error) {
m, err := sub.NextMsg(buscalls.SeatAckWithin)
if err != nil {
return "", nil, err
}
return m.Subject, m.Data, nil
}
ack := func(subject string) error { _, err := nc.Request(subject, nil, 10*time.Second); return err }
if _, err := chunks.Receive(a.Chunks, next, ack, &out); err != nil {
t.Fatal(err)
}
if err := in.Verify(out.Bytes()); err != nil {
t.Fatal(err)
}
return out.Bytes()
}
@@ -1,137 +0,0 @@
package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// saidEvents keeps what a publisher says.
type saidEvents struct {
mu sync.Mutex
said map[string]any
ids map[string]string
heard chan string
}
func newSaidEvents() *saidEvents {
return &saidEvents{said: map[string]any{}, ids: map[string]string{}, heard: make(chan string, 8)}
}
func (s *saidEvents) say(event string, body any, id string) error {
s.mu.Lock()
s.said[event], s.ids[event] = body, id
s.mu.Unlock()
s.heard <- event
return nil
}
// AS5 (§22, §354): the assigner takes mesh-wants' accepted kind `step` for the composition step and says the seat's
// step-ended.<node> with the ask's id, what the step was asked at and what it came to; an ask of another step is left.
func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
said := newSaidEvents()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
var asked meshwants.StepRun
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
t.Fatal("a step ask of another step is taken")
}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
if c.take(context.Background(), "", body) {
t.Fatal("a step ask with no id is taken")
}
if !c.take(context.Background(), "ask-1", body) {
t.Fatal("the composition step is not taken")
}
select {
case <-said.heard:
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
event := meshwants.EventStepEnded + ".n1"
end, ok := said.said[event].(meshwants.StepEnded)
if !ok || said.ids[event] != "ask-1" {
t.Fatalf("said %v under %v, want step-ended.n1 under the ask's id", said.said, said.ids)
}
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
t.Fatalf("step-ended %+v", end)
}
}
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
// asker, the phase the run is in and the time, and says none once it has ended.
func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said := newSaidEvents()
tick := make(chan time.Time)
stopped := make(chan struct{})
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
release := make(chan struct{})
// A-294: the asker is the one caller mesh-wants' definition lists for step, read from the seat list; none while unread
// or while it lists more than one.
runner := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}}
two := seatdef.Definition{Seat: meshwants.Seat, Accepts: []seatdef.Message{{Name: meshwants.AcceptStep,
Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner", "seat:mesh-catalog"}}}}}
for _, d := range []struct {
def seatdef.Definition
read bool
want string
}{{runner, true, "mesh-pipeline-runner"}, {runner, false, ""}, {two, true, ""}} {
if got := stepAskerOf(func() (seatdef.Definition, bool) { return d.def, d.read })(); got != d.want {
t.Errorf("the asker of %+v (read %v) = %q, want %q", d.def.Accepts, d.read, got, d.want)
}
}
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
asker: stepAskerOf(func() (seatdef.Definition, bool) { return runner, true }),
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d != time.Minute {
t.Errorf("the progress ticks every %s, want a minute", d)
}
return tick, func() { close(stopped) }
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main"})
if !c.take(context.Background(), "ask-1", body) {
t.Fatal("the composition step is not taken")
}
progress := meshwants.EventStepProgress + ".n1"
for range 2 {
tick <- at
if e := <-said.heard; e != progress {
t.Fatalf("said %s on a tick, want %s", e, progress)
}
}
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
t.Fatalf("step-progress %+v", p)
}
close(release)
if e := <-said.heard; e != meshwants.EventStepEnded+".n1" {
t.Fatalf("said %s at the end, want step-ended", e)
}
select {
case <-stopped:
default:
t.Fatal("the ticker is not stopped when the step has ended")
}
}
@@ -1,451 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// A delivery is waited for mesh-wide (design 47 G §376, §389, A-189, A-246). At its start the assigner reads whole, from their first
// message, the streams of the catalog seat's version-released and of its own version-delivered (mesh-sdk process.FromFirst, A-233),
// and the watchman's machine-away and machine-back, never its state; then it hears them. A released version of a module is said
// version-delivered on the seat mesh-wants once, when every present machine placing it on the branch it is released from runs, healthy,
// the want the assigner wrote naming that version; a version no present machine places is not delivered. A dependent whose want
// changes fingerprint waits, by an edge of kind 2, 4 or 6, for its dependency's newest released version not yet said delivered, on
// the branch it is released from; on another branch no version is released and nothing waits. The wait is said wait-held, step
// delivery, bound 60 minutes from the version's release, kept in the state delivery-waits under the dependent's name, and said
// wait-ended when the version is delivered (ran) or released by hand (assigner_release, released).
// deliveryWaitBound is a dependent's delivery wait's bound, from its dependency's version-released.
const deliveryWaitBound = 60 * time.Minute
// Wait-ended's how of a delivery wait.
const (
deliveryWaitRan = "ran"
deliveryWaitReleased = "released"
)
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
type releaseHeard struct {
Version string
Parts map[string]string
Source int64
At time.Time
}
// newestOf is the newest version of module released, the last heard of it in the stream's order.
func (r *releasedVersions) newestOf(module string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
h, ok := r.newest[module]
return h, ok
}
// deliveredVersions are the version-delivered said, the assigner's own, by module and version: each is said once.
type deliveredVersions struct {
mu sync.Mutex
said map[string]bool
}
func (d *deliveredVersions) key(module, version string) string { return module + "@" + version }
// put takes one version-delivered in.
func (d *deliveredVersions) put(data []byte) error {
var e meshwants.VersionDeliveredEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-delivered that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-delivered naming no module or version: %s", data)
}
d.mark(e.Module, e.Version)
return nil
}
func (d *deliveredVersions) mark(module, version string) {
d.mu.Lock()
defer d.mu.Unlock()
if d.said == nil {
d.said = map[string]bool{}
}
d.said[d.key(module, version)] = true
}
// has says whether version-delivered of module's version was said.
func (d *deliveredVersions) has(module, version string) bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.said[d.key(module, version)]
}
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
func presenceOf(data []byte) (string, time.Time, error) {
var e wd.MachineAwayEvent // machine-back has the same fields
if err := json.Unmarshal(data, &e); err != nil {
return "", time.Time{}, err
}
if e.Machine == "" {
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
}
return e.Machine, e.At, nil
}
// hearDelivery reads whole, then follows, the four streams a delivery is judged from: version-released, version-delivered,
// machine-away and machine-back. A stream not heard is said as the assigner's finding, and what it carries is not known until
// the assigner starts again.
func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
for _, s := range []struct {
stream, words string
take func([]byte) error
}{
{meshcatalog.StreamVersionReleased, "no want names a released version and no delivery is judged", w.released.put},
{meshwants.StreamVersionDelivered, "a version said delivered before is said again once it is judged", w.delivered.put},
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, true, at)
}
return err
}},
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, false, at)
}
return err
}},
} {
take := s.take
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
if err := take(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf, process.FromFirst())
if err != nil {
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
continue
}
nh.Heard(s.stream)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
}
// releaseBranch is the branch module's versions are released from: the first watched branch of its registration (§371).
func (p *passer) releaseBranch(module string) string {
if p.ends.following() {
return p.ends.branchOf(module, "")
}
return p.w.trunkName()
}
// placing is every machine placing module on the branch its versions are released from, sorted.
func (p *passer) placing(module string) []string {
branch := p.releaseBranch(module)
var out []string
for node, placed := range p.w.placedBranches()[module] {
if p.ends.following() {
placed = p.ends.branchOf(module, placed)
} else {
placed = p.w.trunkName() // the trunk-only form: every placement runs the trunk's
}
if placed == branch {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// deliveryStanding is where module's version stands: the present placing machines not yet running it healthy as its want names
// it, and the placing machines away.
func (p *passer) deliveryStanding(module, version string) (notRunning, away []string, placing int) {
gone := p.w.awayNow()
n := assign.Nodes{Running: p.w.runningNow()}
wanted := p.w.wantsOf()[module]
nodes := p.placing(module)
for _, node := range nodes {
if gone[node] {
away = append(away, node)
continue
}
want, ok := wanted[node]
if !ok || want.Removed || want.Version != version || !n.RunsHealthy(node, module, want.Fp) {
notRunning = append(notRunning, node)
}
}
return notRunning, away, len(nodes)
}
// judgeDelivered says version-delivered of each module's newest released version not said yet, once every present machine placing
// it runs it healthy; judged again at each pass, which each running, machine-away, machine-back and placement change wakes.
func (p *passer) judgeDelivered() {
for _, module := range p.releasedModules() {
rel, _ := p.w.released.newestOf(module)
if p.w.delivered.has(module, rel.Version) {
continue
}
notRunning, away, placing := p.deliveryStanding(module, rel.Version)
if placing == 0 || len(away) == placing || len(notRunning) > 0 {
continue
}
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s is not said, and is judged again at the next pass: %v", module, rel.Version, err)
}
}
}
// sayDelivered says version-delivered of module's release rel, with the machines away, and keeps it said.
func (p *passer) sayDelivered(module string, rel releaseHeard, away []string) error {
at := p.clock().UTC()
body := meshwants.VersionDeliveredEvent{V: keys.Version, Module: module, Version: rel.Version, Parts: rel.Parts, Away: away, At: &at}
if err := p.wantsSay(meshwants.EventVersionDelivered, body, fmt.Sprintf("%s.%s.%s.%s", meshwants.EventVersionDelivered, module, rel.Version, at.Format(time.RFC3339Nano))); err != nil {
return err
}
p.w.delivered.mark(module, rel.Version)
return nil
}
// releasedModules is every module a version-released was heard of, sorted.
func (p *passer) releasedModules() []string {
r := &p.w.released
r.mu.Lock()
defer r.mu.Unlock()
out := make([]string, 0, len(r.newest))
for m := range r.newest {
out = append(out, m)
}
sort.Strings(out)
return out
}
// deliveryWait is what a dependent waits for: its dependency, the version and the machines not yet running it.
type deliveryWait struct {
dependency string
rel releaseHeard
notRunning []string
away []string
}
// waitsForDelivery says whether module, whose want on its nodes changes to fp on branch, waits for a dependency's delivery: by
// each edge of kind 2, 4 or 6 into it in branch's catalog half, the dependency's newest released version on branch not yet said
// delivered. A want that changes no fingerprint waits for nothing; on a branch no version is released from, nothing waits.
func (p *passer) waitsForDelivery(module, fp, branch string, nodes []string) (deliveryWait, bool) {
wanted := p.w.wantsOf()[module]
changes := false
for _, node := range nodes {
if w, ok := wanted[node]; !ok || w.Removed || w.Fp != fp {
changes = true
}
}
if !changes || fp == "" {
return deliveryWait{}, false
}
var deps []string
for _, e := range p.w.edgesOf(branch, module) {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
dep, _, _ := strings.Cut(e.From, ":")
if dep != "" && dep != module && !slices.Contains(deps, dep) {
deps = append(deps, dep)
}
}
sort.Strings(deps)
for _, dep := range deps {
if p.releaseBranch(dep) != branch {
continue
}
rel, ok := p.w.released.newestOf(dep)
if !ok || p.w.delivered.has(dep, rel.Version) {
continue
}
notRunning, away, _ := p.deliveryStanding(dep, rel.Version)
return deliveryWait{dependency: dep, rel: rel, notRunning: notRunning, away: away}, true
}
return deliveryWait{}, false
}
// words is a delivery wait in plain words, as assigner_why_not and the status view say it.
func (d deliveryWait) words() string {
if len(d.notRunning) == 0 && len(d.away) > 0 {
return fmt.Sprintf("waiting for %s to be back", strings.Join(d.away, ", "))
}
return fmt.Sprintf("waiting for delivery of %s %s on %s", d.dependency, d.rel.Version, strings.Join(d.notRunning, ", "))
}
// deliveryWaitBody is the wait-held of module's delivery wait d.
func deliveryWaitBody(module, fp string, d deliveryWait, since time.Time) meshwants.Wait {
from := d.rel.At.UTC()
if from.IsZero() {
from = since.UTC()
}
return meshwants.Wait{Step: meshwants.WaitStepDelivery, Owner: meshwants.Seat, Module: module, Version: fp, Part: d.dependency,
On: d.dependency + "@" + d.rel.Version, Machines: d.notRunning, Since: since.UTC(), ClockFrom: &from, Bound: deliveryWaitBound.String(),
Words: d.words()}
}
// deliveryWaiting is one delivery wait the state delivery-waits keeps, by its dependent's name: the wait-held said, and a release
// by hand written by assigner_release.
type deliveryWaiting struct {
meshwants.Wait
// Released is set by assigner_release: the wait ends released and the dependent's want is written.
Released bool `json:"released,omitempty"`
}
// saidDeliveryWaits is every delivery wait the state delivery-waits holds, by module; a value that does not decode is said and left.
func (p *passer) saidDeliveryWaits() map[string]deliveryWaiting {
out := map[string]deliveryWaiting{}
for key, value := range p.w.read(buckets.DeliveryWaits()) {
var w deliveryWaiting
if err := json.Unmarshal(value, &w); err != nil || w.Module == "" {
p.logf("%s %s holds no delivery wait, left: %v", buckets.DeliveryWaits(), key, err)
continue
}
out[w.Module] = w
}
return out
}
// holdForDelivery says whether module's want, changing to fp on branch, is held for a dependency's delivery this pass: it says
// wait-held once and keeps it; a wait released by hand holds nothing; a wait that no longer holds is ended ran.
func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string, nodes []string) bool {
if p.kvs[buckets.DeliveryWaits()] == nil {
return false
}
key := keys.EscapeName(module)
said, was := p.saidDeliveryWaits()[module]
d, waits := p.waitsForDelivery(module, fp, branch, nodes)
switch {
case was && said.Released:
p.endDeliveryWait(ctx, key, said, deliveryWaitReleased) // the want is written now, so it changes nothing at the next pass
return false
case !waits:
if was {
p.endDeliveryWait(ctx, key, said, deliveryWaitRan)
}
return false
}
body := deliveryWaitBody(module, fp, d, p.clock())
if was && said.On == body.On && said.Version == fp && slices.Equal(said.Machines, body.Machines) {
return true
}
if was {
body.Since = said.Since
}
id := fmt.Sprintf("%s.delivery.%s.%s.%s", key, fp, body.On, strings.Join(body.Machines, ","))
if err := p.wantsSay(meshwants.EventWaitHeld, body, id); err != nil {
p.logf("the delivery wait of %s is not said, and is said at the next pass: %v", module, err)
return true
}
kept := deliveryWaiting{Wait: body}
p.put(ctx, buckets.DeliveryWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
return true
}
// endDeliveryWait says wait-ended of a delivery wait with how and removes it from the state.
func (p *passer) endDeliveryWait(ctx context.Context, key string, w deliveryWaiting, how string) {
body := w.Wait
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+".delivery."+body.Version+"."+how); err != nil {
p.logf("the end of the delivery wait of %s is not said, and is said at the next pass: %v", w.Module, err)
return
}
p.remove(ctx, buckets.DeliveryWaits(), key)
}
// againAsk is one ask of assigner_delivered_again, kept in the state delivered-again under its module (A-250): the version to say
// delivered again, who asked and why, and, while it does not hold, why not.
type againAsk struct {
Module string `json:"module"`
Version string `json:"version"`
By string `json:"by"`
Why string `json:"why"`
Waits string `json:"waits,omitempty"`
}
// releaseOf is the release heard of module's version.
func (r *releasedVersions) releaseOf(module, version string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
if h, ok := r.newest[module]; ok && h.Version == version {
return h, true
}
for key, k := range r.seen {
if strings.HasPrefix(key, module+"@") && k.Version == version {
return releaseHeard{Version: version, Parts: k.Parts}, true
}
}
return releaseHeard{}, false
}
// deliveredAgain takes each ask of assigner_delivered_again, heard at once through the watch of the state delivered-again: it says
// version-delivered again when every present machine placing the module runs that version healthy and removes the ask, else keeps
// it with why it does not hold, which assigner_waits reads.
func (p *passer) deliveredAgain(ctx context.Context) {
if p.kvs[buckets.DeliveredAgain()] == nil {
return
}
for key, value := range p.w.read(buckets.DeliveredAgain()) {
var a againAsk
if err := json.Unmarshal(value, &a); err != nil || a.Version == "" {
p.logf("%s %s holds no ask, left: %v", buckets.DeliveredAgain(), key, err)
continue
}
module := a.Module
if module == "" {
module = key
}
waits := ""
rel, ok := p.w.released.releaseOf(module, a.Version)
notRunning, away, placing := p.deliveryStanding(module, a.Version)
switch {
case !ok:
waits = fmt.Sprintf("no version-released of %s %s is heard", module, a.Version)
case placing == 0:
waits = fmt.Sprintf("no machine places %s", module)
case len(away) == placing:
waits = fmt.Sprintf("every machine placing %s is away: %s", module, strings.Join(away, ", "))
case len(notRunning) > 0:
waits = fmt.Sprintf("%s %s does not run healthy on %s", module, a.Version, strings.Join(notRunning, ", "))
}
if waits == "" {
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s asked again is not said, and is asked at the next pass: %v", module, a.Version, err)
continue
}
p.logf("version-delivered of %s %s said again, asked by %s: %s", module, a.Version, a.By, a.Why)
p.remove(ctx, buckets.DeliveredAgain(), key)
continue
}
if a.Waits != waits {
a.Waits = waits
p.put(ctx, buckets.DeliveredAgain(), key, func() ([]byte, error) { return json.Marshal(a) })
}
}
}
@@ -1,188 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §376, §389, §498, A-189, A-246: a released version is said version-delivered once, when every present machine placing
// it runs healthy the want naming it, a machine away named and not waited for, none when every placing machine is away; a dependent
// whose want changes fingerprint waits by an edge of kind 2, 4 or 6 for its dependency's newest released version not yet said
// delivered, said wait-held (step delivery, 60 minutes from the release) once and kept in delivery-waits, and wait-ended ran once
// the version is delivered; a kind-1 edge, or a dependency with no version released, waits for nothing; one released by hand ends
// released and its want is written.
func TestADependentWaitsForItsDependencysDelivery(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
waits := &memKV{values: map[string][]byte{}}
again := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.DeliveryWaits(): waits, buckets.DeliveredAgain(): again}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
p.w.runningHeard = true
placeOn(t, p.w, "ace", "db", "app")
placeOn(t, p.w, "g14", "db")
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
want := func(node, module, fp, version string) {
key, _ := mwc.WantKey(node, module)
raw, err := mwc.EncodeWant(mwc.Want{Fp: fp, Version: version, Self: true})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Wants(), key, raw)
}
runs := func(node, module, fp string) {
raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "fingerprint": fp, "at": at})
if err := p.w.hearRunning("running", raw); err != nil {
t.Fatal(err)
}
}
source := int64(3)
release := func(module, version string) {
source++
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{V: 1, Module: module, Version: version, Source: source, At: &at,
Artifacts: []meshcatalog.VersionArtifact{{Artifact: "program", Digest: "sha256:d"}}})
if err := p.w.released.put(raw); err != nil {
t.Fatal(err)
}
}
saidOf := func(event string) []said {
var out []said
for _, e := range events {
if e.event == event {
out = append(out, e)
}
}
return out
}
want("ace", "app", fp("1"), "")
// No version of db is released: app's new fingerprint waits for nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app waits with no version of db released")
}
release("db", "1.1.0")
want("ace", "db", fp("d"), "1.1.0")
want("g14", "db", fp("d"), "1.1.0")
runs("ace", "db", fp("d"))
// A kind-1 edge orders nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 1, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("a kind-1 edge holds app")
}
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
// app's want unchanged waits for nothing.
if p.holdForDelivery(ctx, "app", fp("1"), trunkBranch, []string{"ace"}) {
t.Fatal("app's unchanged want waits")
}
// g14 places db and does not run 1.1.0 yet: not delivered, and app's new version waits, said once.
p.judgeDelivered()
if !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) || !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.1.0")
}
if len(saidOf(meshwants.EventVersionDelivered)) != 0 {
t.Fatalf("db is said delivered while g14 does not run it: %+v", events)
}
held := saidOf(meshwants.EventWaitHeld)
if len(held) != 1 {
t.Fatalf("wait-held said %d times, want once", len(held))
}
var w meshwants.Wait
_ = json.Unmarshal(held[0].body, &w)
if w.Step != meshwants.WaitStepDelivery || w.Part != "db" || w.Module != "app" || w.Version != fp("2") || w.Bound != "1h0m0s" ||
w.ClockFrom == nil || !w.ClockFrom.Equal(at) || len(w.Machines) != 1 || w.Machines[0] != "g14" || w.Words != "waiting for delivery of db 1.1.0 on g14" {
t.Errorf("the wait said is %+v", w)
}
if _, ok := waits.values["app"]; !ok {
t.Errorf("the wait is not kept in delivery-waits: %v", waits.values)
}
// g14 away: every present machine runs it, so it is delivered, g14 named away; once only.
p.w.heardPresence("g14", true, at)
p.judgeDelivered()
p.judgeDelivered()
delivered := saidOf(meshwants.EventVersionDelivered)
var d meshwants.VersionDeliveredEvent
if len(delivered) != 1 || json.Unmarshal(delivered[0].body, &d) != nil || d.Module != "db" || d.Version != "1.1.0" ||
len(d.Away) != 1 || d.Away[0] != "g14" || d.Parts["db:program"] != "sha256:d" {
t.Fatalf("version-delivered said %+v", delivered)
}
// The wait ends ran, and app's want goes on.
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app still waits once db 1.1.0 is delivered")
}
ended := saidOf(meshwants.EventWaitEnded)
if len(ended) != 1 || json.Unmarshal(ended[0].body, &w) != nil || w.How != deliveryWaitRan {
t.Errorf("wait-ended said %+v", ended)
}
if _, ok := waits.values["app"]; ok {
t.Error("the ended wait is kept")
}
// A newer release with every placing machine away is not delivered, and its wait says the machines it waits to be back.
release("db", "1.2.0")
p.w.heardPresence("ace", true, at)
p.judgeDelivered()
if len(saidOf(meshwants.EventVersionDelivered)) != 1 {
t.Error("a version is said delivered while every placing machine is away")
}
if !p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.2.0")
}
held = saidOf(meshwants.EventWaitHeld)
if json.Unmarshal(held[len(held)-1].body, &w) != nil || w.Words != "waiting for ace, g14 to be back" {
t.Errorf("the wait while every machine is away says %q", w.Words)
}
// Released by hand (assigner_release writes it): the wait ends released and app's want is written.
var kept deliveryWaiting
_ = json.Unmarshal(waits.values["app"], &kept)
kept.Released = true
raw, _ := json.Marshal(kept)
p.w.put(buckets.DeliveryWaits(), "app", raw)
if p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("a released wait holds app")
}
ended = saidOf(meshwants.EventWaitEnded)
if json.Unmarshal(ended[len(ended)-1].body, &w) != nil || w.How != deliveryWaitReleased {
t.Errorf("the released wait ended %+v", w)
}
// Presence: the later of machine-away and machine-back holds, whichever stream is read first.
p.w.heardPresence("ace", false, at.Add(time.Minute))
p.w.heardPresence("ace", true, at)
if p.w.awayNow()["ace"] || !p.w.online()["ace"] {
t.Error("an older machine-away read after a newer machine-back leaves ace away")
}
// A-250: an ask of assigner_delivered_again of 1.1.0, which ace (present) runs and g14 (away) does not, is said again and taken
// away; one of 1.2.0, which no machine runs, is kept with why.
ask := func(version string) {
raw, _ := json.Marshal(againAsk{Module: "db", Version: version, By: "op", Why: "the catalog missed it"})
p.w.put(buckets.DeliveredAgain(), "db", raw)
}
before := len(saidOf(meshwants.EventVersionDelivered))
ask("1.1.0")
p.deliveredAgain(ctx)
if len(saidOf(meshwants.EventVersionDelivered)) != before+1 || p.w.has(buckets.DeliveredAgain(), "db") {
t.Errorf("the ask of 1.1.0 said %d version-delivered and is kept %v", len(saidOf(meshwants.EventVersionDelivered))-before, p.w.has(buckets.DeliveredAgain(), "db"))
}
ask("1.2.0")
p.deliveredAgain(ctx)
var a againAsk
if json.Unmarshal(p.w.value(buckets.DeliveredAgain(), "db"), &a) != nil || a.Waits != "db 1.2.0 does not run healthy on ace" ||
len(saidOf(meshwants.EventVersionDelivered)) != before+1 {
t.Errorf("the ask of 1.2.0 = %+v", a)
}
}
@@ -1,82 +0,0 @@
package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
)
// runtimeObjects is the kept data assignment-revisions as the bundle reaches it through its runtime (the old objects
// section of a foundation part, design 47 lines 1362-1366).
type runtimeObjects struct{ o busstate.Objects }
func (r runtimeObjects) Put(_ context.Context, name string, data []byte) error {
_, err := r.o.Put(name, data, "")
return err
}
func (r runtimeObjects) Get(_ context.Context, name string) ([]byte, error) {
d, err := r.o.Get(name)
if err != nil {
return nil, err
}
if d.Info.Name == "" || d.Info.Deleted {
return nil, assigngraph.ErrNoObject
}
return d.Data, nil
}
func (r runtimeObjects) Names(context.Context) ([]string, error) {
infos, err := r.o.List()
if err != nil {
return nil, err
}
names := []string{}
for _, i := range infos {
if !i.Deleted {
names = append(names, i.Name)
}
}
return names, nil
}
// graphVerbs are the seat mesh-wants' verbs revision, entry and difference (design 47 lines 1365-1366: an assignment graph
// revision is served by the seat's verbs), each reading only the kept revisions.
func graphVerbs(o assigngraph.Objects) map[string]func(map[string]any) (any, error) {
return map[string]func(map[string]any) (any, error){
meshwants.VerbRevision: func(a map[string]any) (any, error) {
var in meshwants.RevisionInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Revision(context.Background(), o, in)
},
meshwants.VerbEntry: func(a map[string]any) (any, error) {
var in meshwants.EntryInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Entry(context.Background(), o, in)
},
meshwants.VerbDifference: func(a map[string]any) (any, error) {
var in meshwants.DifferenceInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Difference(context.Background(), o, in)
},
}
}
// decodeArgs reads a verb's arguments into its input type; a caller's own fields beside them are the runtime's.
func decodeArgs(a map[string]any, into any) error {
raw, err := json.Marshal(a)
if err != nil {
return err
}
return json.Unmarshal(raw, into)
}
@@ -1,82 +0,0 @@
package main
import (
"context"
"fmt"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
f, is := g.objects.(assigngraph.Filler)
if !is {
return 0, 0, false
}
used, capacity, err := f.Fill(ctx)
return used, capacity, err == nil && capacity > 0
}
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
at := p.clock().UTC()
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
}
}
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
// its cap), so a fill reached while the process was down is said too.
func (p *passer) fillAtStart(ctx context.Context) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return
}
switch {
case used >= capacity:
g.full = true
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
case used*5 >= capacity*4:
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
}
}
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
// 80 % again. It answers true while the store is full.
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return false
}
if g.full && used*5 < capacity*4 {
g.full = false
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
}
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
g.full = false
return false
}
if !g.full {
g.full = true
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
}
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
return true
}
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
func (p *passer) fillingAfter(ctx context.Context, before int64) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
}
}
@@ -1,83 +0,0 @@
package main
import (
"context"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
type filled struct {
objs map[string][]byte
base int64
cap int64
}
func (f *filled) Put(_ context.Context, name string, data []byte) error {
f.objs[name] = data
return nil
}
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := f.objs[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (f *filled) Names(context.Context) ([]string, error) {
out := []string{}
for n := range f.objs {
out = append(out, n)
}
return out, nil
}
func (f *filled) Fill(context.Context) (int64, int64, error) {
used := f.base
for _, d := range f.objs {
used += int64(len(d))
}
return used, f.cap, nil
}
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
ctx := context.Background()
p := newPasser(kvSet{}, time.Now())
var events []string
p.publishWants = func(event string, body any, id string) error {
if _, ok := body.(meshwants.Fill); ok {
events = append(events, event)
}
return nil
}
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
if len(events) != 0 {
t.Fatalf("events %v", events)
}
o.base = 700
p.graph.started = false
p.keepGraph(ctx)
if len(events) != 1 || events[0] != meshwants.EventFilling {
t.Fatalf("at the start past 80 %%: %v", events)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
o.base = 990
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
t.Fatalf("a revision that does not fit: events %v", events)
}
p.fullBefore(ctx, c)
if n := len(events); n != 2 {
t.Errorf("full is said once: %v", events)
}
o.base = 100
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
t.Errorf("room: events %v", events)
}
}
@@ -1,192 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
// from the catalog halves it follows, the placements, the first branch of each registration and each node's capability reading,
// keeps it in the objects assignment-revisions and says the seat mesh-wants' event assignment-revision; the wants of the pass
// that follows are written from the revision it holds.
type graphKeeper struct {
objects assigngraph.Objects
// halves is every followed branch's catalog graph revision held now.
halves func() map[string]mcc.Half
// last is the revision last computed, so a pass whose inputs did not change keeps and says nothing.
last meshwants.AssignmentRevision
has bool
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
kept meshwants.AssignmentRevision
source int64
ok bool
said string
// cycled is the cycles last said (assigngraph.Cycles).
cycled string
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
full, started bool
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
pins *pinKeeper
// computed is the revision computed by the last pass, with its entries: the edges the wants wait on (actingwaits.go).
computed assigngraph.Computed
}
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
func (p *passer) keepGraph(ctx context.Context) {
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return
}
c := assigngraph.Compute(p.graphInputs())
g.computed = c
if p.ends != nil {
p.ends.setRunner(assigngraph.Runner(slices.Collect(maps.Values(c.Entries)))) // the runner's member of a group goes first (A-334)
}
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
p.logf("the assignment graph leaves out %s", u)
}
}
if cycles := strings.Join(c.Cycles, "; "); cycles != g.cycled {
g.cycled = cycles
if cycles != "" {
p.logf("the assignment graph holds a cycle among the acting waits, within one step or across steps: %s", cycles)
}
}
if !g.started {
g.started = true
p.fillAtStart(ctx)
}
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
return
}
used, _, _ := g.fillOf(ctx)
if p.fullBefore(ctx, c) {
return
}
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
if err != nil {
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
return
}
g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true
g.source = 0
for _, cat := range rev.Catalogs {
if cat.Branch == p.w.trunkName() {
g.source = cat.Source
}
}
if !fresh {
return
}
p.fillingAfter(ctx, used)
if g.pins != nil {
g.pins.judge() // the revision before ceased to be current
}
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
}
}
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
// node's capability reading and the first branch of each registration.
func (p *passer) graphInputs() assigngraph.Inputs {
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
// (design 47 G §708).
if r := p.seats.Reader(); r != nil {
in.Definitions = r.Definitions()
}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
first := p.w.trunkName()
in.FirstBranch = func(string) string { return first }
if p.ends != nil {
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
}
return in
}
// capabilityDigests is each node's capability reading, by node: the statement its engine made of what it reads, and the
// capabilities its tool runner states with the fingerprint of the runner that announced them, as the node's record carries
// them (design 46 approval mark §6, lead answer A-203), the digest over all of it.
func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem {
records := p.w.records()
known := p.w.knownNodes()
slices.Sort(known)
var out []meshwants.AssignmentRevisionCapabilitiesItem
for _, node := range known {
read := p.reads != nil && p.reads.Known(node)
rec, recorded := records[node]
if !read && !recorded {
continue
}
var reads []string
if read {
reads = slices.Clone(p.reads.Reads([]string{node})[node])
slices.Sort(reads)
}
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
Fingerprint: rec.RunnerFingerprint}
slices.Sort(item.Readings)
raw, _ := json.Marshal(struct {
Reads []string `json:"reads"`
Readings []string `json:"readings"`
Fingerprint string `json:"fingerprint"`
}{reads, item.Readings, item.Fingerprint})
sum := sha256.Sum256(raw)
item.Digest = "sha256:" + hex.EncodeToString(sum[:])
out = append(out, item)
}
return out
}
// sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is
// said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there.
func (p *passer) sayWanted(node, module, fp string, removed bool) {
if p.publishWants == nil {
return
}
g := p.graph
if g == nil || !g.ok {
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
return
}
at := p.clock().UTC()
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at}
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, g.kept.Revision, at.Format(time.RFC3339Nano))
if err := p.wantsSay(event, body, id); err != nil {
p.logf("wanted of %s on %s is not said: %v", module, node, err)
}
}
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
// version and the parts of the `version-released` whose source is the one the kept assignment graph revision was computed from (A-67).
func (p *passer) namedBuild(module, trunkFP string) func(fp string) (assign.KeptBuild, bool) {
return func(fp string) (assign.KeptBuild, bool) {
if k, ok := p.w.ends.partsOf(module, fp); ok {
return k, true
}
if fp != trunkFP || p.graph == nil || !p.graph.ok {
return assign.KeptBuild{}, false
}
return p.w.released.of(module, p.graph.source)
}
}
@@ -1,134 +0,0 @@
package main
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// said is one event the passer said on the seat mesh-wants.
type said struct {
event, id string
body json.RawMessage
}
// Design 47 G A-67, §292: the pass keeps an assignment graph revision per catalog revision taken in, before it writes any want, and
// says assignment-revision; A-61, §693: a want written or removed is said `wanted` on the subject of its node and module in the same
// write, from the kept revision and its catalog's source, a put that wrote nothing says nothing, and with no revision kept none is said.
func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
kv := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): kv}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
placeOn(t, p.w, "ace", "web")
// Design 46 approval mark §6: a node's reading keeps the fingerprint of the runner that announced it, as its record says.
rec, _ := json.Marshal(nodeengine.HaveByNodeNodeValue{V: 1, Record: &nodeengine.NodeRecord{V: 1, Node: "ace",
RunnerCapabilities: []string{"bus-state-calls", "approval-mark/1"}, RunnerFingerprint: "sha256:runner"}})
p.w.put(buckets.Have, "ace.node", rec)
if c := p.capabilityDigests(); len(c) != 1 || c[0].Node != "ace" || c[0].Fingerprint != "sha256:runner" ||
!slices.Equal(c[0].Readings, []string{"approval-mark/1", "bus-state-calls"}) || !strings.HasPrefix(c[0].Digest, "sha256:") {
t.Errorf("the capability readings are %+v", c)
}
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fpTest}}}
objects := memObjects{}
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} }}
want := mwc.Want{Fp: fpTest, Self: true}
p.putWant(ctx, "ace", "web", want) // no revision kept yet: written, and not said
if len(events) != 0 || kv.puts != 1 {
t.Fatalf("with no revision kept: %d events, %d puts", len(events), kv.puts)
}
key, _ := mwc.WantKey("ace", "web")
delete(p.w.read(buckets.Wants()), key) // the world forgets the want, as before the first pass
kv.puts = 0
p.w.put(buckets.Wants(), key, nil)
p.keepGraph(ctx)
if len(events) != 1 || events[0].event != meshwants.EventAssignmentRevision || len(objects) == 0 {
t.Fatalf("the pass kept %d objects and said %+v", len(objects), events)
}
p.keepGraph(ctx) // the same inputs: nothing kept, nothing said
if len(events) != 1 {
t.Fatalf("a pass of the same inputs said %d events", len(events))
}
p.putWant(ctx, "ace", "web", want)
p.putWant(ctx, "ace", "web", want) // unchanged: not written, not said
if len(events) != 2 || kv.puts != 1 {
t.Fatalf("a want put twice: %d events, %d puts", len(events), kv.puts)
}
if wantEvent := "wanted.ace.web"; events[1].event != wantEvent {
t.Errorf("wanted said on %q, want %q", events[1].event, wantEvent)
}
var w meshwants.WantedEvent
if err := json.Unmarshal(events[1].body, &w); err != nil || w.Node != "ace" || w.Module != "web" || w.Fp != fpTest || w.Removed || w.Revision != 1 || w.Source != 9 {
t.Errorf("wanted = %+v (%v)", w, err)
}
p.putWant(ctx, "ace", "web", mwc.Want{Removed: true})
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
}
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
// keeps none, and the run's group-ended drops it.
before, puts := len(events), kv.puts
ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5}
var asked []meshcatalog.RevisionInput
call := func(key string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.RevisionInput)
asked = append(asked, in)
rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}}
a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}}
return json.Marshal(a)
}
if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 {
t.Errorf("a try's candidate asked %v: %v", asked, err)
}
if err := p.keepCandidate(ctx, call, ev); err != nil {
t.Fatal(err)
}
if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 {
t.Errorf("the candidate's revision was asked as %+v", asked)
}
a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"})
if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 {
t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err)
}
if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the run's entry of web is %+v: %v", e.Entry, err)
}
if len(events) != before || kv.puts != puts {
t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts)
}
if err := p.dropCandidate(ctx, "run-7"); err != nil {
t.Fatal(err)
}
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok {
t.Error("the run's group-ended left its candidate kept")
}
if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil {
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
}
}
@@ -1,163 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"slices"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
)
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the
// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name
// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each
// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings;
// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
// half, never its own output back (§195).
type halves struct {
w *world
call mcc.Call
reader *views.ProcessReader
wake func()
logf func(string, ...any)
ctx context.Context
branches *mcc.Branches
mu sync.Mutex
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
settled chan<- struct{}
told bool
}
// follow makes the branch follower on the reader; the reader's start asks the first views.
func (h *halves) follow() error {
b, err := mcc.FollowBranchesProcess(h.reader, h.call)
if err != nil {
return err
}
b.Logf = h.logf
b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
}
b.Adopted = h.adopted
h.mu.Lock()
h.branches = b
h.mu.Unlock()
return nil
}
// first is the first watched branch followed, the trunk's: "" while none is.
func (h *halves) first() string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return ""
}
if f := b.Followed(); len(f) > 0 {
return f[0]
}
return ""
}
// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its
// first view.
func (h *halves) adopted(branch string, c *mcc.Catalog) {
first := h.first()
if first == "" {
return
}
h.w.setTrunk(first)
h.wake()
if branch != first || h.settled == nil {
return
}
h.mu.Lock()
told := h.told
h.told = true
h.mu.Unlock()
if told {
return
}
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
}()
}
// changed takes one entry of branch's half into the world, or a module taken out of it.
func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) {
defer h.wake()
if deleted {
h.w.dropModule(branch, module)
return
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(raw, &e); err != nil {
h.logf("the entry of %s on branch %q does not read: %v", module, branch, err)
return
}
v, n, m := fromEntry(e)
h.w.putModule(branch, module, v, n, m)
h.w.putEdges(branch, module, e.Edges)
}
// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of
// its words.
func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) {
v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages,
Path: e.Path, Repository: e.Repository, Verdict: e.Verdict}
n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires,
Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
for p := range e.Packages {
n.Packages = append(n.Packages, p)
}
slices.Sort(n.Packages)
if len(e.Manifest) == 0 {
return v, n, nil
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return v, n, nil
}
sum := sha256.Sum256(raw)
v.Manifest = "sha256:" + hex.EncodeToString(sum[:])
man, err := catalogue.Decode(raw)
if err != nil {
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
}
n.Serves = graph.ServesOf(man)
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
return v, n, &parsed{module: e.Module, manifest: man}
}
// views is the catalog graph revision held now of each followed branch whose first view is answered.
func (h *halves) views() map[string]mcc.Half {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
out := map[string]mcc.Half{}
if b == nil {
return out
}
for _, branch := range b.Followed() {
if c, ok := b.Catalog(branch); ok {
if half, ok := c.View(); ok {
out[branch] = half
}
}
}
return out
}
@@ -1,377 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
const inputsTool = "assigner_inputs"
const (
// inputsKeptFor is how long an input is kept at the longest.
inputsKeptFor = 24 * time.Hour
// inputsSwept is how many of the oldest keys each new ask judges.
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the input's own key, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
)
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
// inputsCap is the state inputs' cap (design 47 G §354's kept-data row: 64 MiB, the manifest's max-mib): at it, an ask is refused.
const inputsCap = 64 << 20
// inputKeeper keeps the inputs over the state and its object store.
type inputKeeper struct {
vals stateobjects.Values
now func() time.Time
logf func(string, ...any)
// used is how many bytes the state and its object store hold; nil judges no cap.
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
}
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
if k.used != nil {
used, err := k.used()
if err != nil {
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
digest := digestOf(body)
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
// input a sweep deleted, changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
}
askKey := inputsAskPrefix + keys.EscapeName(ask)
e, err := k.vals.KV.Get(askKey)
if err != nil || e == nil {
return err
}
if err := k.vals.KV.Delete(string(e.Value)); err != nil {
return err
}
return k.vals.KV.Delete(askKey)
}
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
var inputs, asks []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
asks = append(asks, key)
} else {
inputs = append(inputs, key)
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > inputsSwept {
inputs = inputs[:inputsSwept]
}
for _, key := range inputs {
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
continue
}
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
for _, key := range asks {
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
}
}
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none.
func (k *inputKeeper) keyOf(digest string) (string, error) {
all, err := k.vals.KV.Keys()
if err != nil {
return "", err
}
slices.Sort(all)
for _, key := range all {
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
return key, nil
}
}
return "", nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
func (k *inputKeeper) body(digest string) ([]byte, error) {
if len(digest) != 64 {
return nil, errors.New("input takes the input's sha256, 64 hex digits")
}
key, err := k.keyOf(digest)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
}
if key == "" {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
var text string
found, err := k.vals.Get(key, &text)
if err != nil {
return nil, fmt.Errorf("the input %s cannot be read: %w", key, err)
}
if !found {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
b, err := base64.StdEncoding.DecodeString(text)
if err != nil || digestOf(b) != digest {
return nil, fmt.Errorf("the input %s does not hold the bytes its digest names", key)
}
return b, nil
}
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
// verb answers the seat's verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
// size. A caller that does not acknowledge within `within` ends the send, said in logf.
func (k *inputKeeper) verb(line buscalls.ChunkLine, within time.Duration) process.SeatHandler {
return func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.InputInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
if !callToken.MatchString(in.Call) {
return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
}
digest := strings.TrimPrefix(strings.TrimSpace(in.Digest), "sha256:")
body, err := k.body(digest)
if err != nil {
return nil, err
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: in.Call}
total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
go func() {
if err := send.Send(line, body, within, k.logf); err != nil {
k.logf("the send %s of the input %s: %v", in.Call, digest, err)
}
}()
return meshwants.InputAnswer{Digest: digest, Chunks: total}, nil
}
}
// inputObjects is the object store beside the state inputs (`inputs-objects`, a state's own, §712), opened when first used: until
// the grants part has made it every call is process.ErrNotYet.
type inputObjects struct {
js jetstream.JetStream
module string
mu sync.Mutex
bus stateobjects.Objects
}
func (o *inputObjects) open() (stateobjects.Objects, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.bus != nil {
return o.bus, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object store of the state inputs is not open: %w", err)
}
o.bus = stateobjects.BusObjects{Store: store}
return o.bus, nil
}
func (o *inputObjects) MostBytes() int { return 0 }
func (o *inputObjects) Put(name string, data []byte) error {
b, err := o.open()
if err != nil {
return err
}
return b.Put(name, data)
}
func (o *inputObjects) Get(name string) ([]byte, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.Get(name)
}
func (o *inputObjects) Delete(name string) error {
b, err := o.open()
if err != nil {
return err
}
return b.Delete(name)
}
func (o *inputObjects) List() ([]stateobjects.Object, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.List()
}
// inputsOver is the inputs the program keeps, over its bucket and its object store beside it; nil when the bucket is not opened.
func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, logf func(string, ...any)) *inputKeeper {
if kv == nil {
return nil
}
objects := &inputObjects{js: js, module: module}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
used := func() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
n := int64(st.Bytes())
objs, err := objects.List()
if err != nil && !errors.Is(err, process.ErrNotYet) {
return 0, err
}
for _, o := range objs {
n += o.Size
}
return n, nil
}
return &inputKeeper{vals: v, now: time.Now, logf: logf, used: used}
}
// inputSaid is one kept input as assigner_inputs lists it.
type inputSaid struct {
Key string `json:"key"`
Time string `json:"time"`
Digest string `json:"digest"`
Size int `json:"size"`
Input string `json:"input,omitempty"`
}
// inputsRead is the tool assigner_inputs over the kept inputs: every kept input with its size, or the one named by digest whole.
func inputsRead(k *inputKeeper) stdio.Tool {
return stdio.Tool{Name: inputsTool, Description: "The inputs the assigner keeps for the composition steps it asked a build seat to run (design 47 G §354): each " +
"with its write time, digest and size, or one by `digest` whole (base64). Read-only.",
Input: schema(map[string]string{"digest": "one input's sha256"}, nil), Run: func(args map[string]any) (any, error) {
want := arg(args, "digest")
all, err := k.vals.KV.Keys()
if err != nil {
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
}
slices.Sort(all)
out := []inputSaid{}
for _, key := range all {
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
continue
}
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
}
said := inputSaid{Key: key, Time: t, Digest: d, Size: base64.StdEncoding.DecodedLen(len(text))}
if b, err := base64.StdEncoding.DecodeString(text); err == nil {
said.Size = len(b)
}
if want != "" {
said.Input = text
}
out = append(out, said)
}
return map[string]any{"inputs": out}, nil
}}
}
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if err := k.ended(e.Ask); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
if ends != nil {
ends.heard(e)
}
_ = m.Ack()
}
@@ -1,231 +0,0 @@
package main
import (
"bytes"
"crypto/rand"
"encoding/json"
"errors"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (k *kvMem) Get(key string) (*stateobjects.Entry, error) {
k.mu.Lock()
defer k.mu.Unlock()
v, ok := k.vals[key]
if !ok {
return nil, nil
}
return &stateobjects.Entry{Value: v, Revision: k.revs[key]}, nil
}
func (k *kvMem) Create(key string, v []byte) error {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, 1
return nil
}
func (k *kvMem) Update(key string, v []byte, rev uint64) error {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, rev+1
return nil
}
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
delete(k.vals, key)
return nil
}
func (k *kvMem) Keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
sort.Strings(out)
return out, nil
}
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(string) error { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
func testKeeper(objs *objectsMem, now func() time.Time) *inputKeeper {
return &inputKeeper{vals: stateobjects.Values{KV: newKVMem(), Objects: objs, Store: "inputs-objects"}, now: now, logf: func(string, ...any) {}}
}
// Design 47 G §354, §712, A-237: an input of any size is kept as an object beside the state, and the seat mesh-wants' verb input
// answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, time.Now)
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := k.keep("ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := k.body(digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := k.verb(line, time.Second)
raw, _ := json.Marshal(meshwants.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshwants.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+strings.Repeat("b", 64)+`","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// assigner_inputs lists it with its size and gives one whole by digest
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["inputs"].([]inputSaid)) != 1 || list.(map[string]any)["inputs"].([]inputSaid)[0].Size != len(body) {
t.Errorf("assigner_inputs = %+v %v", list, err)
}
}
// §354: an input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, judged at each new ask.
func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
now := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, func() time.Time { return now })
d1, err := k.keep("ask/one", []byte(`{"heads":"abc"}`))
if err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("a kept input is not read: %v", err)
}
if err := k.ended("ask/other"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("the step-ended of another ask deleted it: %v", err)
}
if err := k.ended("ask/one"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err == nil {
t.Error("the input of an ended ask is still answered")
}
if keys, _ := k.vals.KV.Keys(); len(keys) != 0 {
t.Errorf("keys left after the step-ended: %v", keys)
}
d2, _ := k.keep("ask/two", []byte(`{"heads":"def"}`))
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask/three", []byte(`{"heads":"ghi"}`)); err != nil {
t.Fatal(err)
}
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
}
File diff suppressed because it is too large Load Diff
@@ -8,19 +8,12 @@ package main
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"io"
"log"
"maps"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"os"
"os/signal"
"slices"
"strings"
"syscall"
"time"
@@ -28,26 +21,16 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"git.novox.be/novox/mesh-sdk/go/provider"
)
func main() {
if os.Getenv(process.BrokerFileEnv) == "" {
if os.Getenv("MESH_BROKER_FILE") == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
if err := serveTools(); err != nil {
@@ -67,164 +50,54 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buckets.Grants,
buckets.Have,
buckets.Provisions,
provider.AssignmentsBucket,
delivery.BuildsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
var readFamilies = []string{buckets.ComposedRead}
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-module-assigner's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
const engineReadsState = "engine-reads"
// engineReadsBucket is that state's bucket.
func engineReadsBucket() string { return assignment.Module() + "_" + engineReadsState }
// writtenBucket is the bucket of the state written (pipelines.go): the pipeline ends the wants follow.
func writtenBucket() string { return assignment.Module() + "_" + writtenState }
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
func seatBuckets() []string {
return []string{plc.AssignmentsBucket(), plc.ProposalsBucket(), plc.CandidatesBucket(),
plc.PulsesBucket(), copiesBucket()}
}
// ownBuckets are the assigner's own states, as its manifest declares them.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-module-assigner's assignments, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its assignments by
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546).
var keptStates = []announce.State{
{Name: mma.StateAssignments, Readable: true},
{Name: mma.StateProposals, Readable: true},
{Name: mma.StateCandidates, Readable: true},
{Name: mma.StatePulses, Readable: true},
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: assign.FindingsState, Readable: true},
{Name: buckets.PoliciesState, Readable: true},
{Name: mma.StateSeatHolders, Readable: true},
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
{Name: views.StatusState, Readable: true}, // its first views of the indexer's states (views.go)
{Name: engineReadsState, Readable: true},
{Name: retiredState}, // read only by the copy into assignments (migrate.go)
{Name: copiesState, Readable: true},
{Name: writtenState, Readable: true},
{Name: buckets.WaitsState, Readable: true},
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
}
// kvSet is every bucket the assigner opened, by name.
type kvSet map[string]jetstream.KeyValue
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (kvSet, error) {
out := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == buckets.Puts() && err != nil {
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if slices.Contains(seatBuckets(), name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without assignments no module is placed.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == plc.SeatHoldersBucket() && err != nil {
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
// holds which seat here, and the controller's seats still answer.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == buckets.Provisions && err != nil {
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
// until then no want waits for a provision, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.PoliciesBucket() && err != nil {
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
// person's policy is read, so the controller's beside each composition holds, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.FindingsBucket() && err != nil {
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
// router and kept as no finding.
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if name == engineReadsBucket() && err != nil {
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
// assigner_engine_reads lists nothing.
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
continue
}
if name == buckets.Inputs() && err != nil {
// New with the verb input (design 47 G §354): only the tool process reads it, so the program starts without it.
logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err)
continue
}
if name == pinsBucket() && err != nil {
// New with the pins (design 47 G §579): without it no pin is kept and no revision is removed.
logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err)
continue
}
if name == buckets.Waits() && err != nil {
// New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged
// as before.
logf("the bucket %s cannot be opened (%v): no outside wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveredAgain() && err != nil {
// New with A-250: without it assigner_delivered_again is not taken until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no version is said delivered again until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveryWaits() && err != nil {
// New with A-246: without it no dependent waits for its dependency's delivery until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no delivery wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.PutWaits() && err != nil {
// New with A-211: without it no put's wait is said until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no put's wait is said until the assigner starts again", name, err)
continue
}
if name == retiredBucket() && err != nil {
// The retired state is read only for the copy into `assignments` (migrate.go): a mesh made after the switch has
// none, and nothing is copied.
logf("the bucket %s cannot be opened (%v): nothing is copied from it", name, err)
continue
}
if name == buckets.Grants && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
continue
}
if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
}
@@ -233,30 +106,36 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
return out, nil
}
// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket(), buckets.Provisions, assign.PoliciesBucket(), assign.FindingsBucket(),
buckets.Puts(), engineReadsBucket(), retiredBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket():
return true
}
return slices.Contains(seatBuckets(), name) || name == buckets.Grants
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
account, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
}
account, err := process.ReadAccount(path)
if err != nil {
return err
}
plc.SetModule(account.Module) // the assigner names its own states from its credential's module (A-296, §428)
conn, js, err := account.Dial()
opts := []nats.Option{
nats.Name(account.Node + "." + account.Module),
nats.UserInfo(account.User, account.Password),
nats.CustomInboxPrefix(account.InboxPrefix()),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(account.Fingerprint) != "" {
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
}
conn, err := nats.Connect(account.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
kvs, err := open(ctx, js, logger.Printf)
if err != nil {
return err
@@ -271,130 +150,37 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := said.Keeps(keptStates...); err != nil {
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
findings, writer := findingsOf(ctx, account, conn, js, said, kvs, logger.Printf)
if writer != nil {
defer writer.Close()
}
// Each change of the states its readers read is said on its views family (ADR 0615 §5, ADR 0619 §2); a watch of it
// that ends stops the program, as a watch of what a pass reads does.
viewErrs := make(chan error, len(viewedStates()))
startViews(ctx, js, conn, account, kvs, wireOf(writer), viewErrs, logger.Printf)
go func() {
select {
case <-ctx.Done():
case err := <-viewErrs:
logger.Printf("%v: the assigner stops, and is started again", err)
conn.Close()
}
}()
iv := &catalogViews{js: js, account: account, call: said.Call, writer: wireOf(writer), logf: logger.Printf,
status: process.AwaitBucket(ctx, js, conn, firstViewsBucket(), "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)}
// The inputs of the composition steps it asks a build seat to run: the seat mesh-wants' verb input is answered here in chunks
// of the core connection, of any size (design 47 G §354, §712, A-237).
inputs := inputsOver(kvs[buckets.Inputs()], js, account.Module, logger.Printf)
// The seat list (seatlist.go), followed in follow over the connection and the hub: the verb input's callers are checked
// against the seat's definition it holds (A-186).
gate := seatlist.NewPart("assigner", seatListAgainCallers)
if inputs != nil {
stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)})
if err != nil {
return fmt.Errorf("answering the verb input: %w", err)
}
defer stopInput()
}
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, inputs, gate, logger, getenv)
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, account process.Account, conn *nats.Conn, js jetstream.JetStream, said *process.Conn, kvs kvSet, logf func(string, ...any)) (assign.Findings, *heldWriter) {
var f assign.Findings
if kvs[assign.FindingsBucket()] == nil {
return f, nil
}
state := said.State(assign.FindingsState)
w, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
Module: account.Module, State: state, Node: account.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf})
if err != nil {
logf("the assigner's finding writer cannot be made, so its findings are not written: %v", err)
return f, nil
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
f.Put = func(x mcseat.Finding) error { return w.Write(ctx, x) }
f.Delete = func(key string) error {
scope, rest, _ := strings.Cut(key, ".")
i := strings.LastIndex(rest, ".")
if i < 0 {
return fmt.Errorf("%q is no finding's key", key)
}
return w.Clear(ctx, scope, rest[:i], rest[i+1:], "the assigner no longer observes it")
}
f.Keys = state.Keys
return f, w
}
// heldWriter is the assigner's finding writer: its findings are raised as they happen, so the writer holds them (design 47 G A-29).
type heldWriter = views.HeldWriter[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]
// wireOf is w as the views parts' finding writer; nil when the assigner has none, so no part writes through a nil writer.
func wireOf(w *heldWriter) views.WireWriter {
if w == nil {
return nil
}
return w
}
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
var thisNode string
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
type emitting struct{ *process.Conn }
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
// optionalEnds is what stops when the watch of a bucket that never gates the first pass ends.
func optionalEnds() map[string]string {
return map[string]string{buckets.Puts(): "no put is taken further", buckets.Answers: "no ask is answered"}
}
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
func watched() []string {
return []string{
buckets.Grants,
plc.AssignmentsBucket(),
plc.ProposalsBucket(),
plc.CandidatesBucket(),
plc.PulsesBucket(),
copiesBucket(),
buckets.Have,
buckets.Asks(),
buckets.Differences(),
buckets.Wants(),
assign.PoliciesBucket(),
buckets.Provisions,
plc.SeatHoldersBucket(),
engineReadsBucket(),
writtenBucket(),
buckets.Waits(),
buckets.PutWaits(),
buckets.DeliveryWaits(),
buckets.DeliveredAgain(), // its watch wakes a pass at once on an ask of assigner_delivered_again (A-250)
}
var watched = []string{
provider.AssignmentsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerCompositionsBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *catalogViews, inputs *inputKeeper, gate *seatlist.Part, logger *log.Logger, getenv func(string) string) error {
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
w := newWorld()
// The callers' seat entries are resolved from the assigner's own seat-holders as it last read them (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return w.read(plc.SeatHoldersBucket()), nil }))
dirty := make(chan struct{}, 1)
wake := func() {
select {
@@ -402,26 +188,15 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
default:
}
}
errs := make(chan error, len(watched())+2)
settled := make(chan struct{}, len(watched())+2)
// What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read
// from the node's record where the engine does not answer (the fallback, said in the journal).
reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf}
if bus != nil {
reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1)
reads.Once = bus.CallOn // its fault keeps the fallback at once while the kept call asks on (ADR 0668 §3)
}
if kv := kvs[engineReadsBucket()]; kv != nil {
reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
reads.Delete = func(node string) error { return kv.Delete(ctx, node) }
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
watch := func(name string, opts ...jetstream.WatchOpt) error {
kw, err := kvs[name].WatchAll(ctx, opts...)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == buckets.Puts() || name == buckets.Answers
optional := name == delivery.AssignerPutsBucket
go func() {
defer kw.Stop()
for {
@@ -432,7 +207,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case e, ok := <-kw.Updates():
if !ok {
if optional {
logger.Printf("the watch of %s ended: %s until the assigner starts again", name, optionalEnds()[name])
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
return
}
errs <- fmt.Errorf("the watch of %s ended", name)
@@ -448,11 +223,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
if name == engineReadsBucket() {
reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed
continue
}
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
w.put(name, e.Key(), value)
wake()
}
}
@@ -461,10 +232,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
if kvs[buckets.Puts()] != nil {
if err := watch(buckets.Puts()); err != nil {
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, buckets.Puts())
delete(kvs, delivery.AssignerPutsBucket)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
@@ -473,111 +244,38 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[buckets.Puts()] != nil || time.Since(triedPuts) < PutsReopenEvery {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, buckets.Puts())
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
if err != nil {
return
}
kvs[buckets.Puts()] = kv
if err := watch(buckets.Puts()); err != nil {
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, buckets.Puts())
delete(kvs, delivery.AssignerPutsBucket)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", buckets.Puts())
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
started := 0
for _, name := range watched() {
if kvs[name] == nil && startsWithout(name) {
continue // not on the bus at this start (open said so)
}
started++
for _, name := range watched {
var opts []jetstream.WatchOpt
if name == buckets.Have {
// what each node ran healthy before, for a set-back
opts = append(opts, jetstream.IncludeHistory())
if name == delivery.HaveBucket {
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
}
if err := watch(name, opts...); err != nil {
return err
}
}
// The assigner's one pull of its durable on the shared events stream: the indexer's views and the engines' reads changed
// are handlers of it, started once both are registered (process.EventHub).
hub := process.NewEventHub(logger.Printf)
// The seat list (seatlist.go), read over the connection and heard on the hub, given to catalogue at every change, and a
// pass made then: registered before the hub starts.
if bus != nil {
gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
// §389). Registered before the hub starts.
ends, away := &runEnds{}, &awayNodes{}
if inputs != nil {
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
return err
}
}
for subject, isAway := range map[string]bool{meshwatchdogs.SubjectMachineAwayAll: true, meshwatchdogs.SubjectMachineBackAll: false} {
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPresence(away, isAway, m, logger.Printf) }); err != nil {
logger.Printf("%s is not heard, so a composition step asks a machine that sleeps: %v", subject, err)
}
}
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
// (DLV-F2r, ADR 0631): the first pass waits for the three answers, as it waits for every watch to hand what stands.
if iv != nil {
iv.hub = hub
started += iv.start(ctx, w, wake, settled)
// The teller's answers' picture is watched as a bucket is, but never gates the first pass: a teller that does not
// answer its first view only leaves the asks unanswered, said by the reader's finding (A-175).
if iv.answers != nil {
kvs[buckets.Answers] = iv.answers
if err := watch(buckets.Answers); err != nil {
logger.Printf("%v: no ask is answered until the assigner starts again", err)
}
}
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
// module is placed.
type appeared struct {
name string
kv jetstream.KeyValue
}
assignmentsAppeared := make(chan appeared, len(seatBuckets())+2)
for _, name := range seatBuckets() {
if kvs[name] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-module-assigner waits for it",
logger.Printf).Ready(func(kv jetstream.KeyValue) { assignmentsAppeared <- appeared{name, kv} })
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{buckets.Grants, assign.PoliciesBucket(), buckets.Provisions, plc.SeatHoldersBucket()} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { assignmentsAppeared <- appeared{name, kv} })
}
}
takeAssignments := func(a appeared) error {
if kvs[a.name] != nil {
return nil // bound already, by the other of its two events
}
kvs[a.name] = a.kv
logger.Printf("the bucket %s is open and watched", a.name)
if err := watch(a.name); err != nil {
return err
}
wake() // a state made empty brings no change of its own: the pass copies the retired state into it (migrate.go)
return nil
}
for range started {
for range len(watched) + 1 {
select {
case <-ctx.Done():
return nil
@@ -586,110 +284,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case <-settled:
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
reads: reads, seats: gate}
// The wait on an outside cause (outside.go, design 47 G §706): the keeper's three events say when to judge again, and the
// seat mesh-wants' wait-held and wait-ended are said; registered before the hub starts.
if bus != nil && kvs[buckets.Waits()] != nil {
bound, settle, err := outsideSettings(getenv)
if err != nil {
return err
}
p.outside = newOutsideActs(bus.CallKept, wake, logger.Printf, bound, settle)
for _, subject := range []string{mcseat.SubjectConditionRaised, mcseat.SubjectConditionChanged, mcseat.SubjectConditionCleared} {
subject := subject
if err := hub.Handle(subject, func(m jetstream.Msg) {
p.outside.Heard(m.Subject(), m.Data())
if err := m.Ack(); err != nil {
logger.Printf("acknowledging %s: %v", m.Subject(), err)
}
}); err != nil {
logger.Printf("%s is not heard, so a wait on an outside cause is judged again only when its check changes: %v", subject, err)
}
}
p.outside.readStart()
}
if js != nil {
p.publishWants = wantsPublisher(ctx, js, thisNode)
if iv != nil && iv.h != nil {
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)
p.graph = &graphKeeper{objects: objects, halves: iv.h.views}
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
return p.keepCandidate(ctx, iv.call, ev)
}
iv.candidates.Store(&keep)
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
p.entries = meshcatalogclient.NewEntries(iv.h.call)
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
// on the hub, registered before it starts.
if kv := kvs[pinsBucket()]; kv != nil {
k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf}
if err := k.made(); err != nil {
logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err)
}
if err := hearPins(hub, k); err != nil {
logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err)
} else {
p.graph.pins = k
k.watchUnpins(kv)
}
}
}
}
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
// Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it
// is not there yet, a node is asked again only when it comes back online.
if js != nil {
// The teller's ask-ended (issue 1966): an ask ended at the channel without an answer is written again; registered
// before hearReadsChanged starts the hub.
if err := hearAskEnded(ctx, js, hub, kvs[buckets.Asks()], staleOfWorld(w), logger.Printf); err != nil {
logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err)
}
readEndedAtBind(ctx, js, hub, kvs[buckets.Asks()], staleOfWorld(w), bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf)
if kv := kvs[writtenBucket()]; kv != nil {
// The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow
// once the first registrations are written (§95, §353).
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
p.ends.trunk = w.trunkName
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
err := hearEnds(hub, p.ends)
if err == nil && bus != nil {
err = followRegistrations(ctx, hub, bus.Call, p.ends)
}
if err != nil {
logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
}
}
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
if err := hearTaken(hub, &w.taken, wake, logger.Printf); err != nil {
logger.Printf("the indexer's take-ins are not heard, so an entry held by an order edge waits until the assigner starts again: %v", err)
}
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
hearRunning(ctx, js, w, wake, findings, logger.Printf)
}
if bus != nil && kvs[buckets.Puts()] != nil {
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, call: bus.CallKeptOn}, keeper: inputs, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep)
}
}
if bus != nil {
p.seat = newSeatActs(bus, wake, logger.Printf)
}
if js != nil {
p.publish = seatPublisher(ctx, js, thisNode)
}
wake()
for {
select {
@@ -698,29 +297,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case err := <-errs:
return err
case <-settled:
case kv := <-assignmentsAppeared:
if err := takeAssignments(kv); err != nil {
return err
}
case <-dirty:
reopenPuts(p)
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
// bus now holds, and says the seat waits while one is missing.
var unbound []string
for _, name := range seatBuckets() {
if kvs[name] != nil || js == nil {
continue
}
if kv, err := js.KeyValue(ctx, name); err == nil {
if err := takeAssignments(appeared{name, kv}); err != nil {
return err
}
continue
}
unbound = append(unbound, name)
}
p.sayWaits(unbound)
p.pass(ctx)
}
}
@@ -731,13 +309,13 @@ var PutsReopenEvery = time.Minute
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) error {
return func(c oc.Condition) error {
subject, err := occ.ConditionSubject(assignment.Module())
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(delivery.AssignerModule)
if err != nil {
return err
}
data, err := occ.EncodeCondition(c)
data, err := asks.EncodeCondition(c)
if err != nil {
return err
}
@@ -745,54 +323,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) erro
return err
}
}
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: acknowledges it, keeps what it says
// of the node (a statement older than it does not stand) and asks the node again.
func readsChangedHandler(reads *enginereads.Keeper, logf func(string, ...any)) func(jetstream.Msg) {
return func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
switch {
case !ours:
case err != nil:
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
reads.AskNode(node, enginereads.Event+" that does not read")
default:
// What the event says is kept first: a statement older than it does not stand while the node is asked again.
if e, derr := enginereads.DecodeChanged(m.Data()); derr == nil {
reads.HeardChanged(e)
}
reads.AskNode(node, enginereads.Event)
}
}
}
// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's one hub of its durable
// consumer on EVENTS, and asks the node it names again; the hub also hands the indexer's views to the followers of
// indexerViews, and starts once both are registered.
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *enginereads.Keeper, findings assign.Findings, live func(), logf func(string, ...any)) {
account := process.Account{Node: thisNode, Module: assignment.Module()}
w := notHeardWriter(findings, logf)
handle := readsChangedHandler(reads, logf)
err := hub.Handle(enginereads.SubjectPrefix+"*", handle)
// At each bind every node the assigner holds a statement of is asked again: an event said before its consumer existed
// is not delivered to it (issue 1966 note 155: ace's bootstrapped statement stood; ADR 0668 §4).
hub.OnBound(reads.AtBind(func() []string { return slices.Sorted(maps.Keys(reads.Statements())) }))
if err == nil {
err = hub.Start(ctx, js, account)
}
if err != nil {
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again (a node is asked again when it comes back online)", err)
return
}
w.Heard(process.EventsStream)
if live != nil {
live() // the seat list, unread while the seat was silent, is read now (seatlist.Part.EventsLive)
}
go func() {
<-ctx.Done()
hub.Stop()
}()
}
@@ -1,13 +1,6 @@
package main
import (
"encoding/json"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"context"
"errors"
"fmt"
@@ -23,45 +16,26 @@ import (
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
if viewsEmitter(nil, process.Account{}, nil, &heldWriter{}, t.Logf).Findings == nil {
t.Error("the assigner's views emitter writes no view-unsaid finding")
}
// 1297/B2a: without a credential the program does not start, and says why.
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.Decode(raw)
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
var reads []string
for _, r := range m.Reads {
if slices.Contains(readFamilies, r) {
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
}
reads = append(reads, strings.Replace(r, ".", "_", 1))
}
for _, l := range readFamilies {
if !slices.Contains(m.Reads, l) {
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
}
}
var states []string
for _, s := range m.State {
states = append(states, m.Module+"_"+s.Name)
@@ -69,7 +43,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, pair := range []struct {
what string
code, want []string
}{{"reads", readBuckets, reads}, {"states", append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()), states}} { // the marks are bound by the views family (views.go)
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
if !reflect.DeepEqual(got, want) {
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
@@ -79,7 +53,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, s := range keptStates {
kept = append(kept, m.Module+"_"+s.Name)
}
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()))); !reflect.DeepEqual(got, want) {
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
t.Errorf("the program announces %v, keeps %v", got, want)
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
@@ -88,19 +62,16 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
missing string
err error
starts bool
said string
}{
{buckets.Puts(), jetstream.ErrBucketNotFound, true, "no put is taken further"},
{buckets.Puts(), errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{buckets.Puts(), context.DeadlineExceeded, true, "no put is taken further"},
{buckets.Wants(), errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets())-1 || !strings.Contains(logged, c.said)):
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
@@ -111,21 +82,19 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
placed, err := mmac.AssignmentsValues(mma.Assignments{Node: "ace"})
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
for key, raw := range placed {
kv := kvs[plc.AssignmentsBucket()].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[buckets.Puts()].(*watchKV).closed = true
kvs[buckets.Wants()].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner", seatListAgainCallers), log.New(io.Discard, "", 0), func(string) string { return "" })
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -136,6 +105,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
}
// 1297/B2a: without a credential the program does not start, and says why.
func TestNoCredential(t *testing.T) {
err := run(context.Background(), func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
}
// refusingJS opens every bucket but one, which it refuses with err.
type refusingJS struct {
jetstream.JetStream
@@ -193,19 +170,3 @@ type kvEntry struct {
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (e kvEntry) Revision() uint64 { return 1 }
func (e kvEntry) Created() time.Time { return time.Time{} }
// Keys lists nothing: the retired state this watch stands for holds no key.
func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
return nil, jetstream.ErrNoKeysFound
}
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
return err
}
}
@@ -1,150 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
plc "mesh-delivery-assigner/internal/assignment"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
)
// The state this seat's assignments were kept in before ADR 0641 gave it its name. The assigner reads it here and nowhere
// else: at its start it copies each key into `assignments` (copyRetired) and keeps no reader of it afterwards. The
// manifest still declares the state, so the assigner's account may read it for the copy; a later release drops it.
const retiredState = "placement"
func retiredBucket() string { return plc.Module() + "_" + retiredState }
// CopiedEvent is the module's event saying the copy was made, once: mesh.mod.mesh-delivery-assigner.event.assignments-copied.
const CopiedEvent = "assignments-copied"
// copiedSaid is what the event carries: the keys copied, those already in `assignments` (kept as they are), and the keys of
// the form `placement.<node>` that were dropped (their node key holds the value).
type copiedSaid struct {
Copied int `json:"copied"`
Present int `json:"present"`
Dropped int `json:"dropped"`
Skipped int `json:"skipped"`
}
// copiesState is the assigner's own state of its copies, read by its tool assigner_read_copies, and copiesBucket its bucket.
// The retired state gets nothing written: readers of other modules still parse every key in it.
const copiesState = "copies"
func copiesBucket() string { return plc.Module() + "_" + copiesState }
// copiedMarker is the key, in the state copies, that says the copy ended: its time and the keys it copied. It is written
// after the last key, and the copy is made at every start until it is there and never after, so a copy cut short by a
// crash is completed at the next start and a key `assignments` dropped after the copy ended is never brought back.
const copiedMarker = "copied-from-retired"
type copiedMark struct {
At time.Time `json:"at"`
Copied int `json:"copied"`
}
// copyRetired is the holder's switch from the retired state to `assignments`, with no gap: until the marker is there, it
// creates in `assignments` each key of the retired state in the form `<node>.node` or `<node>.<module>` that is missing
// there (never overwriting one), the module keys first and the node keys, which commit a node, last, and then writes the
// marker. It deletes nothing: the readers delivered before this one still read the retired state, and a delete of a
// `<node>.node` key there is a node leaving (ADR 0641 section 4); the state is dropped in the next release. Keys of the
// form `placement.<node>` are not copied, and a key that fits no form is left and counted. It answers whether the seat may
// go on: a copy that failed is tried again at the next pass.
func (p *passer) copyRetired(ctx context.Context) bool {
if p.copied {
return true
}
old, next, copies := p.kvs[retiredBucket()], p.kvs[plc.AssignmentsBucket()], p.kvs[copiesBucket()]
if next == nil || copies == nil {
return false
}
if old == nil {
p.logf("assignments: the retired state %s cannot be read, so nothing is copied from it", retiredBucket())
p.copied = true
return true
}
switch _, err := copies.Get(ctx, copiedMarker); {
case err == nil:
p.copied = true // the copy ended at an earlier start: nothing is copied again
return true
case !errors.Is(err, jetstream.ErrKeyNotFound):
p.logf("assignments: the marker %s of %s cannot be read, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
names, err := old.Keys(ctx)
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
p.logf("assignments: the keys of %s cannot be listed, and are tried again at the next pass: %v", retiredBucket(), err)
return false
}
// A node's module keys before its node key: the node key is the commit the readers wait for.
slices.SortStableFunc(names, func(a, b string) int {
return strings.Compare(sortForm(a), sortForm(b))
})
said := copiedSaid{}
for _, key := range names {
_, _, kind, perr := mmac.ParseKey(key)
switch {
case strings.HasPrefix(key, retiredState+"."):
// The node named placement could not exist before, so this is the form that held a whole node in one key.
said.Dropped++
case perr != nil || (kind != mmac.NodeKind && kind != mmac.ModuleKind):
said.Skipped++
continue
default:
entry, err := old.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
p.logf("assignments: %s cannot be read, and the copy is tried again at the next pass: %v", key, err)
return false
}
if entry.Operation() != jetstream.KeyValuePut {
continue
}
rev, err := next.Create(ctx, key, entry.Value())
switch {
case errors.Is(err, jetstream.ErrKeyExists):
said.Present++
case err != nil:
p.logf("assignments: %s is not copied, and the copy is tried again at the next pass: %v", key, err)
return false
default:
said.Copied++
p.w.putEntry(plc.AssignmentsBucket(), key, entry.Value(), rev, p.clock())
}
}
}
mark, _ := json.Marshal(copiedMark{At: p.clock(), Copied: said.Copied + said.Present})
if _, err := copies.Put(ctx, copiedMarker, mark); err != nil {
p.logf("assignments: the end of the copy is not written as %s in %s, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
p.copied = true
if said == (copiedSaid{}) {
return true // the retired state holds nothing
}
p.logf("assignments: copied %d keys of %s into %s (%d already there kept as they are, %d of the form placement.<node> not copied, %d fitting no form left)",
said.Copied, retiredBucket(), plc.AssignmentsBucket(), said.Present, said.Dropped, said.Skipped)
if p.seat != nil {
if err := p.seat.bus.Emit(CopiedEvent, said); err != nil {
p.logf("assignments: the copy is not said as an event: %v", err)
}
}
return true
}
// sortForm orders `<node>.<module>` keys before `<node>.node`.
func sortForm(key string) string {
if node, ok := strings.CutSuffix(key, ".node"); ok {
return node + "\x7f"
}
return key
}
@@ -1,118 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assigngraph"
)
// busObjects is the assigner's own object bucket assignment-revisions in the old `objects` section (novox/hq ADR 0619 §2),
// which the grants part creates, as the passer's process reaches it: opened at its first use and kept; until the bucket exists
// every call is an error saying so, and the next call opens it again (design 47 G A-67, modelled on the indexer's busObjects).
type busObjects struct {
js jetstream.JetStream
module string
local string
mu sync.Mutex
o jetstream.ObjectStore
}
// newBusObjects is the object bucket of this local name of the module the account runs as.
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
return &busObjects{js: js, module: cred.Module, local: local}
}
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.o != nil {
return b.o, nil
}
o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(b.module, b.local))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
}
b.o = o
return o, nil
}
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
o, err := b.open(ctx)
if err != nil {
return err
}
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
return err
}
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
data, err := o.GetBytes(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, assigngraph.ErrNoObject
}
return data, err
}
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
infos, err := o.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return []string{}, nil
}
if err != nil {
return nil, err
}
names := make([]string, 0, len(infos))
for _, i := range infos {
names = append(names, i.Name)
}
return names, nil
}
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
func (b *busObjects) Delete(ctx context.Context, name string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
o, err := b.open(ctx)
if err != nil {
return 0, 0, err
}
st, err := o.Status(ctx)
if err != nil {
return 0, 0, err
}
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
if err != nil {
return int64(st.Size()), 0, err
}
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
}
@@ -1,296 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The three order edges of a delivery that crosses a form change (design 47 G §298, A-245, A-258, A-259, A-260). Each is an acting wait of the
// assigner, kept in the state `waits` under `order.<node>.<module>.<part>` beside the acting waits, in the step placing, said as wait-held
// when a want is held for it and wait-ended (ran, released, replaced) when it moves; assigner_waits reads them and assigner_release with
// how `now` releases them, as it does an acting wait. The holders of the seats are read from the seat holders the assignments compose
// (assignment.SeatHoldersOf), never named:
// - the node-backup holder's and the bus module's (the holder of the seat mesh-broker) new versions run before a module whose kept data's
// form changes is moved, since the restore point needs both (OrderBackup);
// - at a revert a node's node-engine goes back last on that node, after every module it restores there runs the reverted build, which it
// starts only once it has said `restored` (OrderEngineLast);
// - the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a source revision at least
// the trunk's head, the highest `src` of the trunk branch's module values (OrderTakenIn, A-260).
// takenSubject is the indexer's event take-in.taken on its own stream, as the manifest's consumes names it.
var takenSubject = process.ModuleStreamSubject(buckets.IndexerModule, "take-in.taken")
// takenSources are the newest source revisions a take-in.taken named, by watched branch, as heard (hearBuildsAndReleases).
type takenSources struct {
mu sync.Mutex
own map[string]int64
}
// put takes one take-in.taken in. A hand drop, and a taken with no source revision (one said before G), take nothing in.
func (t *takenSources) put(data []byte) error {
var e meshcatalog.TakeInTaken
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a take-in.taken that is none: %w", err)
}
if e.Dropped || e.Source == 0 || e.Branch == "" {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.own == nil {
t.own = map[string]int64{}
}
t.own[e.Branch] = max(t.own[e.Branch], e.Source)
return nil
}
// of is the newest source revision of branch taken in; 0 before one is heard.
func (t *takenSources) of(branch string) int64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.own[branch]
}
// orderEdge is one order edge held on module's want on node: part's version fp must run on partNode (OrderBackup, OrderEngineLast), or the
// trunk's source revision source be taken in (OrderTakenIn).
type orderEdge struct {
node, module, part, partNode, fp, entry, order string
source int64
}
func (e orderEdge) key() string { return acting.OrderKey(e.node, e.module, e.part) }
func (e orderEdge) acting() assign.ActingEdge {
return assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace}
}
// orderRuns says an order edge is met: its part runs the version, or the source revision is taken in.
func (p *passer) orderRuns(n assign.Nodes, e orderEdge) bool {
if e.order == acting.OrderTakenIn {
return p.w.taken.of(p.w.trunkName()) >= e.source
}
return n.Runs(e.acting())
}
// seatHolders are the holders of each seat as the assignments compose them, by seat name.
func (p *passer) holdersBySeat() map[string][]mma.SeatHoldersHoldersItem {
out := map[string][]mma.SeatHoldersHoldersItem{}
for _, sh := range assignment.SeatHoldersOf(p.w.assignments()) {
out[sh.Seat] = append(out[sh.Seat], sh.Holders...)
}
return out
}
// orderEdgesOf computes §298's edges on the wants of module m on each of its nodes, into p.orders. wanted is m's wants by node; trunk the
// trunk's module values. A node that has no running build heard, runs m at its trunk version already, or is wanted that version already
// has none: nothing moves. It answers, by node, whether an edge still stands that cannot be recorded as a wait because the catalog entry
// it reads is not answered yet (asked, and its answer wakes a pass): the want is held meanwhile, never written across a form change unread.
func (p *passer) orderEdgesOf(ctx context.Context, m assign.Module, wanted map[string]mwc.Want, trunk map[string]meshcatalog.ModulesByModuleValue) map[string]bool {
unread := map[string]bool{}
p.orderVisited[m.Name] = true
if m.FP == "" || p.graph == nil || p.entries == nil {
return unread
}
var holders map[string][]mma.SeatHoldersHoldersItem
for _, node := range m.Nodes {
if w, ok := wanted[node]; ok && !w.Removed && w.Fp == m.FP {
continue // the version is wanted already: nothing moves, so nothing waits
}
run, heard := p.w.runningOf(node, m.Name)
if !heard || run.Stopped || run.Fingerprint == "" || run.Fingerprint == m.FP {
continue
}
if holders == nil {
holders = p.holdersBySeat()
}
held := func(seatName string) bool {
for _, h := range holders[seatName] {
if h.Module == m.Name && h.Node == node {
return true
}
}
return false
}
add := func(e orderEdge) {
e.node, e.module, e.entry = node, m.Name, m.FP
p.orders[e.key()] = e
}
// 1. a kept data's form changes: the restore point needs the new node-backup holder and bus module.
changes, known := p.formChange(ctx, m.Name, run.Fingerprint, m.FP)
if !known {
unread[node] = true
}
if len(changes) > 0 {
for seatName, onNode := range map[string]bool{catalogue.BackupSeat: true, meshbroker.Seat: false} {
for _, h := range holders[seatName] {
if h.Module == m.Name || (onNode && h.Node != node) || trunk[h.Module].Fp2 == "" {
continue
}
add(orderEdge{part: h.Module, partNode: h.Node, fp: trunk[h.Module].Fp2, order: acting.OrderBackup})
}
}
}
if !held(nodeengine.Seat) && !held(mma.Seat) && !held(meshdelivery.Seat) {
continue
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, m.Name, run.Fingerprint, m.FP)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", m.Name, node, m.FP, err)
continue
}
if !reverted {
continue
}
// 2. at a revert the node-engine goes back last on its node: after every module that is set back there across a form change runs.
if held(nodeengine.Seat) {
for module, byNode := range p.w.wantsOf() {
// What it is set back to is its trunk version, which the rules write as its want in this pass.
target := trunk[module].Fp2
if w, ok := byNode[node]; module == m.Name || !ok || w.Removed || target == "" {
continue
}
xrun, heard := p.w.runningOf(node, module)
if !heard || xrun.Stopped || xrun.Fingerprint == "" || xrun.Fingerprint == target {
continue
}
if back, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, xrun.Fingerprint, target); err != nil || !back {
continue
}
xchanges, known := p.formChange(ctx, module, xrun.Fingerprint, target)
if !known {
unread[node] = true
}
if len(xchanges) > 0 {
add(orderEdge{part: module, partNode: node, fp: target, order: acting.OrderEngineLast})
}
}
}
// 3. the reverted assigner and gate wait for the indexer to take in the trunk's head.
if held(mma.Seat) || held(meshdelivery.Seat) {
var head int64
for _, v := range p.w.trunkOf(p.w.trunkName()) {
head = max(head, v.Src)
}
if head > 0 {
add(orderEdge{part: buckets.IndexerModule, fp: strconv.FormatInt(head, 10), source: head, order: acting.OrderTakenIn})
}
}
}
return unread
}
// orderActing is the order edges of module on node that still hold its want, as acting edges: those not met, and not released by hand. A
// node whose catalog entry is not read yet holds its want too, on a stand-in edge no part runs.
func (p *passer) orderActing(module string, released map[string]bool, unread map[string]bool, n func() assign.Nodes) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
nodes := n()
for _, e := range p.orders {
if e.module == module && e.node == node && !released[e.key()+"@"+e.fp] && !p.orderRuns(nodes, e) {
out = append(out, assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace})
}
}
if unread[node] {
out = append(out, assign.ActingEdge{Node: node, Part: "the catalog entry", Step: seatdef.StepPlace})
}
return out
}
}
// orderWaits keeps the order waits after the pass's wants, as actingWaits keeps the acting waits: it starts the wait of each edge not met,
// ends the wait of one that ran, was replaced or was released, and says each.
func (p *passer) orderWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]mwc.Want) {
if p.kvs[buckets.Waits()] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits())
trunk := p.w.trunk()
if p.orderSaid == nil {
p.orderSaid = map[string]bool{}
}
standing := map[string]bool{}
for key, e := range p.orders {
if w, ok := wanted[e.module][e.node]; ok && !w.Removed && w.Fp == e.entry {
continue
}
if p.orderRuns(n, e) {
continue
}
standing[key] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == e.entry && w.Fp == e.fp {
if !w.Released && !p.orderSaid[key] {
// A wait still held after a restart is said again, as wait-held (A-260).
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry+".again"); err != nil {
p.logf("the wait of %s on %s for %s is not said again, and is said at the next pass: %v", e.module, e.node, e.part, err)
} else {
p.orderSaid[key] = true
}
}
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[e.node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: seatdef.StepPlace, Node: e.node, Module: e.module, Part: e.part, PartNode: e.partNode, Fp: e.fp, Entry: e.entry, Order: e.order,
Since: now, ClockFrom: now, Bound: acting.BoundOf(seat.Edge{Step: seatdef.StepPlace}).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module()), key+"."+e.fp+"."+e.entry); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", e.module, e.node, e.part, err)
continue
}
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
p.orderSaid[key] = true
}
for key, raw := range kept {
if !acting.IsOrder(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
if _, there := trunk[w.Module]; there && !p.orderVisited[w.Module] {
continue // the module was not looked at in this pass (no pipeline passed for it yet): its wait stands as kept
}
how := acting.HowRan
switch e, there := p.orders[key]; {
case w.Released:
how = acting.HowReleased
case there && e.fp != w.Fp, !there && wanted[w.Module][w.Node].Fp != w.Entry:
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module())
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits(), key)
delete(p.orderSaid, key)
}
}
@@ -1,351 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
func orderFP(c string) string {
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
}
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
type orderMod struct {
name string
run, trunk string
forms map[string]int
holds []string
src int64
}
type orderFixture struct {
t *testing.T
p *passer
wants *memKV
events []string
bodies []meshwants.Wait
answers chan struct{}
}
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
t.Helper()
ctx := context.Background()
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
p := f.p
p.logf = t.Logf
p.say = func(oc.Condition) error { return nil }
p.publishWants = func(event string, body any, id string) error {
f.events = append(f.events, event)
f.bodies = append(f.bodies, body.(meshwants.Wait))
return nil
}
digest := "sha256:" + strings.Repeat("c", 64)
pl := mma.Assignments{Node: "ace"}
forms := map[string]int{} // by fingerprint: each module's own
modules := []string{}
for _, m := range mods {
modules = append(modules, m.name)
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
for _, seat := range m.holds {
scope := "node"
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
scope = "mesh"
}
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
}
for fp, form := range m.forms {
forms[fp] = form
}
mdigest := fmt.Sprintf("%064x", len(modules))
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, Src: m.src})
putModuleRaw(p.w, trunkBranch, m.name, mod)
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
endDone(t, p.w, m.name, m.trunk, mdigest)
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
f.runs(m.name, m.run)
wkey, _ := mwc.WantKey("ace", m.name)
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
p.w.put(buckets.Wants(), wkey, old)
f.wants.values[wkey] = old
}
composes(t, p.w, "ace", digest)
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
objects := memObjects{}
for i, at := range history {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, module := range modules {
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
}
placed := mma.Assignments{Node: "ace"}
for _, module := range modules {
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]mma.Assignments{"ace": placed}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
if form := forms[in.Fingerprint]; form > 0 {
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
return f
}
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
func (f *orderFixture) runs(module, fp string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
f.p.w.put(buckets.Have, "ace."+module, have)
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
}
// pass makes a pass, and after it every catalog entry it asked is answered.
func (f *orderFixture) pass() {
f.t.Helper()
f.p.wants(context.Background())
asked := 0
for _, row := range f.p.entries.Rows() {
if row.State == meshcatalogclient.EntryAsked {
asked++
}
}
for range asked {
select {
case <-f.answers:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
f.t.Fatal("an entry asked was not answered")
}
}
}
// wantAt is the fingerprint module is wanted at on ace.
func (f *orderFixture) wantAt(module string) string {
f.t.Helper()
key, _ := mwc.WantKey("ace", module)
w, err := mwc.DecodeWant(f.wants.values[key])
if err != nil {
f.t.Fatal(err)
}
return w.Fp
}
func (f *orderFixture) waits() []string {
var out []string
for key := range f.p.w.read(buckets.Waits()) {
if acting.IsOrder(key) {
out = append(out, key)
}
}
return out
}
// Design 47 G §298 first edge, A-258, A-259: a module whose kept data's form changes is not moved until the new version of the node-backup
// holder and of the holder of the seat mesh-broker run; the wait is kept under `order.` in the state waits, said as wait-held, and ends
// ran (wait-ended) when the holders run; a module whose kept data keeps its form is moved at once.
func TestAModuleWhoseKeptDataChangesFormWaitsForTheNewBackupHolderAndTheBusModule(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "plain", run: orderFP("c"), trunk: orderFP("d"), forms: map[string]int{orderFP("c"): 1, orderFP("d"): 1}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
})
f.pass() // the entries are asked: no want crosses a form change unread
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web is wanted at %s before the entries are read", got)
}
f.pass()
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web moved across a form change at %s while the new node-backup holder and bus module do not run", got)
}
if got := f.wantAt("plain"); got != orderFP("d") {
t.Errorf("a module whose kept data keeps its form is wanted at %s, not moved", got)
}
if len(f.waits()) != 2 || len(f.events) != 2 || f.events[0] != meshwants.EventWaitHeld || f.bodies[0].Step != seatdef.StepPlace || f.bodies[0].Module != "web" {
t.Fatalf("waits %v events %v bodies %+v: want two wait-held in the step place, `order.ace.web.backup` and `order.ace.web.bus`", f.waits(), f.events, f.bodies)
}
if _, err := acting.Decode(f.p.w.read(buckets.Waits())[acting.OrderKey("ace", "web", "backup")]); err != nil {
t.Fatalf("the wait is not kept under its key: %v", err)
}
f.pass()
if len(f.events) != 2 {
t.Fatalf("a pass that changed nothing said %v", f.events)
}
f.runs("backup", orderFP("f"))
f.pass()
if got := f.wantAt("web"); got != orderFP("a") || len(f.events) != 3 || f.events[2] != meshwants.EventWaitEnded || f.bodies[2].How != acting.HowRan {
t.Fatalf("with the bus module still old: web at %s, events %v", got, f.events)
}
f.runs("bus", orderFP("h"))
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 4 {
t.Fatalf("with both running: web at %s, waits %v, events %v", got, f.waits(), f.events)
}
}
// §298 first edge, the hand: assigner_waits reads an order wait and assigner_release with how `now` lets the module move.
func TestAnOrderWaitIsReadAndReleasedByTheExistingTools(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
})
f.pass()
f.pass()
var acts []string
s := newPutTools(memStore{waitsState: f.p.kvs[buckets.Waits()].(*memKV), buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
got, err := s.doWaits(map[string]any{"module": "web"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).Key != "order.ace.web.backup" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "web", "how": "now", "why": "the restore point is taken by hand"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
f.p.w.put(buckets.Waits(), "order.ace.web.backup", f.p.kvs[buckets.Waits()].(*memKV).values["order.ace.web.backup"])
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || f.bodies[len(f.bodies)-1].How != acting.HowReleased {
t.Fatalf("after the release: web at %s, last %+v", got, f.bodies[len(f.bodies)-1])
}
}
// §298 second edge, A-259: at a revert the node-engine goes back last on its node, after every module that is set back there across a
// form change runs the reverted build.
func TestAtARevertTheNodeEngineGoesBackLast(t *testing.T) {
old, now := map[string]string{"web": orderFP("a"), "engine": orderFP("c")}, map[string]string{"web": orderFP("b"), "engine": orderFP("d")}
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("b"), trunk: orderFP("a"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "engine", run: orderFP("d"), trunk: orderFP("c"), holds: []string{nodeengine.Seat}},
}, old, now)
f.pass()
f.pass()
if f.wantAt("web") != orderFP("a") {
t.Fatalf("web is wanted at %s: the module set back is not held", f.wantAt("web"))
}
if got := f.wantAt("engine"); got != orderFP("d") {
t.Fatalf("the node-engine is set back to %s before web runs its reverted build", got)
}
if w := f.waits(); len(w) != 1 || w[0] != "order.ace.engine.web" || f.bodies[len(f.bodies)-1].Module != "engine" {
t.Fatalf("waits %v bodies %+v", w, f.bodies)
}
f.runs("web", orderFP("a"))
f.pass()
f.pass()
if got := f.wantAt("engine"); got != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("with web back: the node-engine is wanted at %s, waits %v", got, f.waits())
}
}
// §298 third edge, A-259, A-260: the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a
// source revision at least the trunk's head (the highest src of the trunk's module values).
func TestTheRevertedAssignerAndGateWaitForTheIndexerToTakeInTheTrunksHead(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a"), "gate": orderFP("c"), "idx": orderFP("e")}, map[string]string{"asg": orderFP("b"), "gate": orderFP("d"), "idx": orderFP("f")}
f := newOrderFixture(t, []orderMod{
{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 6},
{name: "gate", run: orderFP("d"), trunk: orderFP("c"), holds: []string{meshdelivery.Seat}, src: 7},
{name: "idx", run: orderFP("e"), trunk: orderFP("e"), src: 7},
}, old, now)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("b") || f.wantAt("gate") != orderFP("d") {
t.Fatalf("the reverted assigner (%s) and gate (%s) are wanted before the head is taken in", f.wantAt("asg"), f.wantAt("gate"))
}
if len(f.waits()) != 2 || f.bodies[0].Part != "mesh-delivery-indexer" {
t.Fatalf("waits %v bodies %+v", f.waits(), f.bodies)
}
taken := func(branch string, source int64) {
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: branch, Source: source})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
}
taken("beta", 9) // another branch, and
taken(trunkBranch, 6)
f.pass()
if f.wantAt("asg") != orderFP("b") {
t.Fatal("a source revision before the trunk's head was taken in, and the assigner was wanted")
}
taken(trunkBranch, 7)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || f.wantAt("gate") != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("after the head is taken in: asg %s gate %s waits %v", f.wantAt("asg"), f.wantAt("gate"), f.waits())
}
}
// A-260: an order wait still held after a restart is said again as wait-held, by a pass that remembers nothing, and ends at the next
// take-in.taken that names the trunk's head.
func TestAnOrderWaitHeldAfterARestartIsSaidAgain(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a")}, map[string]string{"asg": orderFP("b")}
f := newOrderFixture(t, []orderMod{{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 7}}, old, now)
f.pass()
f.pass()
if len(f.events) != 1 {
t.Fatalf("events %v", f.events)
}
f.p.orderSaid = nil // a process started again
f.pass()
if len(f.events) != 2 || f.events[1] != meshwants.EventWaitHeld || len(f.waits()) != 1 {
t.Fatalf("events %v waits %v", f.events, f.waits())
}
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: trunkBranch, Source: 8})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || len(f.waits()) != 0 || f.events[len(f.events)-1] != meshwants.EventWaitEnded {
t.Fatalf("asg at %s, waits %v, events %v", f.wantAt("asg"), f.waits(), f.events)
}
}
@@ -1,499 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The wait on an outside cause (design 47 G §706, ADR 0677): the assigner judges a node's health after a want moves, and a moved
// module's failing check on a machine whose open conditions name an outside cause is held as a wait (internal/outside has the
// rules). The conditions come only through the seat mesh-conditions: its verb `conditions`, read at each judging as a kept
// call, and its events, which only say when to judge again.
// conditionsVerb is the seat verb the keeper answers `conditions` by.
const conditionsVerb = "seat:" + mcseat.Seat + "." + mcseat.VerbConditions
// wantsPublisher says one of the seat mesh-wants' events, `wait-held` and `wait-ended` among them, under a message id the bus
// keeps one of.
func wantsPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatdef.EventSubject(meshwants.Seat, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", assignment.Module())
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
return err
}
}
// outsideActs keeps, while the process runs, what the judging of outside waits needs beside the kept waits: the conditions read
// for each machine (one answer is one judging), the machines a heard event says to judge again, the machines whose own network
// is held open, and each check's last signature.
type outsideActs struct {
mu sync.Mutex
call func(key string, body any) (json.RawMessage, error) // a kept call
wake func()
logf func(string, ...any)
// bound and settle are the settings outside-wait-minutes and outside-settle-seconds.
bound, settle time.Duration
// reading is a read in flight by machine, answered the answers it brought, not yet judged by.
reading map[string]bool
answered map[string][]outside.Cond
// dirty is a machine whose waits are judged again, cleared a wait's key whose cause's clearing was heard.
dirty map[string]bool
cleared map[string]bool
// sigs is each check's last signature, by node.module.
sigs map[string]string
// own is the machines whose own network is open; ownRead says the read at the start was answered.
own map[string]bool
ownRead bool
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
causes map[string][]string
// started says the first pass judged every wait again.
started bool
}
func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func(), logf func(string, ...any), bound, settle time.Duration) *outsideActs {
if bound <= 0 {
bound = outside.DefaultBound
}
if settle <= 0 {
settle = outside.DefaultSettle
}
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, own: map[string]bool{}, causes: map[string][]string{}}
}
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
// each read at the start from the file the manifest declares with the content ${config:<key>}, which the environment names
// (A-112, as the builder reads its own); zero for one whose file is missing or empty, which is its default, and an error
// naming the file for a content that is no positive whole number.
func outsideSettings(getenv func(string) string) (bound, settle time.Duration, err error) {
minutes, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_WAIT_FILE"))
if err != nil {
return 0, 0, err
}
seconds, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"))
if err != nil {
return 0, 0, err
}
return time.Duration(minutes) * time.Minute, time.Duration(seconds) * time.Second, nil
}
// settingFile reads one setting's file: 0 for no path, no file or an empty one.
func settingFile(path string) (int, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
}
return n, nil
}
// readStart reads the open conditions of every machine once at the start, for the machines whose own network is held.
func (o *outsideActs) readStart() {
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine"})
if err != nil {
o.logf("the open conditions are not read at the start (%v): no machine is held for its own network until its event is heard", err)
return
}
conds, err := outside.ParseConditions(answer)
if err != nil {
o.logf("%v", err)
return
}
o.mu.Lock()
for m := range outside.OwnNetwork(conds) {
o.own[m] = true
}
o.ownRead = true
o.mu.Unlock()
o.wake()
}()
}
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again and keeps the
// machines whose own network is open; nothing is kept of the condition.
func (o *outsideActs) Heard(subject string, body []byte) {
var e mcseat.ConditionEvent
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
o.logf("%s holds what is not a condition's event, and is dropped: %v", subject, err)
return
}
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
o.mu.Lock()
if p := strings.Split(e.Key, "."); len(p) == 3 && p[0] == "machine" && p[2] == "network" {
o.own[p[1]] = !cleared
}
if cleared {
for _, node := range o.causes[e.Key] {
o.dirty[node] = true
o.cleared[node+"|"+e.Key] = true
}
} else if e.Subject != nil {
o.dirty[e.Subject.Machine] = true
for _, m := range e.Subject.Also {
o.dirty[m] = true
}
}
o.mu.Unlock()
o.wake()
}
// Held says whether machine's own network is open: no want moves to it.
func (o *outsideActs) Held(machine string) bool {
if o == nil {
return false
}
o.mu.Lock()
defer o.mu.Unlock()
return o.own[machine]
}
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
// which it starts when none is in flight.
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
o.mu.Lock()
defer o.mu.Unlock()
if c, ok := o.answered[machine]; ok {
delete(o.answered, machine)
return c, true
}
if o.reading[machine] {
return nil, false
}
o.reading[machine] = true
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine", "machine": machine})
var conds []outside.Cond
if err == nil {
conds, err = outside.ParseConditions(answer)
}
o.mu.Lock()
o.reading[machine] = false
if err != nil {
o.logf("the conditions of %s are not read (%v): the move waits on %s", machine, err, outside.OnUnread)
} else {
o.answered[machine] = conds
o.dirty[machine] = true // the kept call answered: the move is judged by it
}
o.mu.Unlock()
if err == nil {
o.wake()
}
}()
return nil, false
}
// outsideReview is the judging of module's check at fp on each of its nodes (§706): it returns the nodes as the rules read them,
// the have of a waiting move no failure (its health unknown) and a go-on's counted healthy, and keeps the waits, says their
// events, finding and ask. moved says the delivery moved owner to node.
func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []string, n assign.Nodes, keeperSelf bool, moved func(owner, node string) bool) assign.Nodes {
o := p.outside
if o == nil || fp == "" || p.kvs[buckets.Waits()] == nil {
return n
}
hex, err := catalogkey.KeyPart(fp)
if err != nil {
return n
}
now := p.clock()
kept := p.w.read(buckets.Waits())
override := map[string]string{}
// A wait of another fingerprint of this module is replaced by the newer want.
for key, raw := range kept {
if w, err := outside.Decode(raw); err == nil && w.Module == module && w.FP != fp {
p.outsideEnd(ctx, key, w, outside.HowReplaced, now)
}
}
for _, node := range nodes {
key := outside.Key(node, module, hex)
var prev *outside.Wait
if raw := kept[key]; raw != nil {
if w, err := outside.Decode(raw); err == nil {
prev = &w
}
}
h, ok := n.Haves[node][module]
sig := fmt.Sprintf("%s|%s|%s", h.Fp, h.Phase, h.Health)
o.mu.Lock()
first := !o.started
changed := o.sigs[node+"."+module] != sig
o.sigs[node+"."+module] = sig
dirty := o.dirty[node]
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
o.mu.Unlock()
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
if !failing && !healthy && prev == nil {
continue
}
if prev != nil && p.outsideAnswered(prev, node, module, fp) {
dirty = true
}
due := prev != nil && (prev.Release != "" || (prev.Settling && !now.Before(prev.SettleEnds)))
var d outside.Decision
if prev != nil && !(first || changed || dirty || due) {
// nothing to judge by: the wait stands as kept; the bound's ask is made when it has run
d = outside.Decision{Wait: prev, Waiting: prev.Standing() || prev.Settling, Own: prev.Decided == outside.HowFailed || prev.Decided == outside.HowOwnFailure,
Counted: prev.Decided == outside.HowGoOn}
if prev.Standing() && !now.Before(prev.ClockFrom.Add(o.bound)) {
d.Ask = true
}
} else {
f := outside.Facts{Now: now, Failing: failing, Healthy: healthy, Node: node, KeeperSelf: keeperSelf, Bound: o.bound, Settle: o.settle,
Moved: func(owner string) bool { return moved(owner, node) }, CauseCleared: cleared}
if failing && !keeperSelf {
f.Conds, f.Read = o.take(node)
}
d = outside.Judge(prev, node, module, fp, f)
o.mu.Lock()
delete(o.dirty, node)
if prev != nil {
delete(o.cleared, node+"|"+prev.Cause)
}
o.mu.Unlock()
}
switch {
case d.Waiting:
override[node] = nodeengineclient.HealthUnknown
case d.Counted:
override[node] = nodeengineclient.HealthOK
}
p.outsideApply(ctx, key, hex, node, module, fp, prev, d, now)
}
o.mu.Lock()
o.started = true
o.mu.Unlock()
if len(override) == 0 {
return n
}
out := n
out.Haves = map[string]map[string]nodeengine.HaveByNodeModuleValue{}
for node, byModule := range n.Haves {
out.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
for m, h := range byModule {
out.Haves[node][m] = h
}
}
for node, health := range override {
h := out.Haves[node][module]
h.Health = health
if health == nodeengineclient.HealthOK {
h.Phase = nodeengineclient.PhaseRunning
}
out.Haves[node][module] = h
}
return out
}
// outsideAnswered reads the operator's answer to the wait's ask, from the teller's record of it under the ask's key (answeredWith,
// §700's path as the assigner reads an ask's answer), and notes it on the wait as a release; it says whether it did.
func (p *passer) outsideAnswered(w *outside.Wait, node, module, fp string) bool {
if w.Ask == "" || w.Release != "" {
return false
}
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return false
}
for _, how := range []string{assign.OptionGoOn, assign.OptionFailed} {
if p.answeredWith(key, how) {
w.Release, w.By, w.Why = how, "the operator", "the answer to the ask "+w.Ask
return true
}
}
return false
}
// outsideApply keeps what a judging came to: the wait's record, its events, its finding and its ask.
func (p *passer) outsideApply(ctx context.Context, key, hex, node, module, fp string, prev *outside.Wait, d outside.Decision, now time.Time) {
o := p.outside
switch {
case d.End != "":
p.outsideEnd(ctx, key, *prev, d.End, now)
if d.Wait != nil {
p.keepWait(ctx, key, *d.Wait)
}
return
case d.Wait == nil:
if prev != nil {
p.outsideEnd(ctx, key, *prev, "", now)
}
return
}
w := *d.Wait
if d.Held {
body := p.outsideBody(w)
if err := p.wantsSay(meshwants.EventWaitHeld, body, key+"."+w.On+"."+w.ClockFrom.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the wait of %s on %s is not said, and is said at the next pass: %v", module, node, err)
return
}
p.outsideFinding(w, true)
}
if d.Ask && w.Ask == "" {
if id := p.outsideAsk(ctx, node, module, fp, w, now); id != "" {
w.Ask = id
}
}
if w.Standing() || w.Settling || w.Decided != "" {
o.mu.Lock()
if w.Cause != "" && w.Standing() {
if !containsString(o.causes[w.Cause], node) {
o.causes[w.Cause] = append(o.causes[w.Cause], node)
}
}
o.mu.Unlock()
if w.Settling {
p.wakeAt(w.SettleEnds, now)
} else if w.Standing() {
p.wakeAt(w.ClockFrom.Add(o.bound), now)
}
p.keepWait(ctx, key, w)
}
}
func containsString(l []string, s string) bool {
for _, x := range l {
if x == s {
return true
}
}
return false
}
// keepWait writes the wait's record.
func (p *passer) keepWait(ctx context.Context, key string, w outside.Wait) {
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return outside.Encode(w) })
}
func (p *passer) outsideBody(w outside.Wait) meshwants.Wait {
since, from := w.Since, w.ClockFrom
return meshwants.Wait{Step: meshwants.WaitStepOutside, Owner: assignment.Module(), Node: w.Node, Module: w.Module, Fingerprint: w.FP, On: w.On,
Since: since, ClockFrom: &from, Bound: w.Bound,
Words: fmt.Sprintf("%s on %s waits on %s", w.Module, w.Node, w.On)}
}
// outsideEnd ends a wait: its wait-ended with how (none for a settle, which was never said), its finding and ask taken away, and
// its record deleted unless the end decided it (the caller keeps that record).
func (p *passer) outsideEnd(ctx context.Context, key string, w outside.Wait, how string, now time.Time) {
if w.Standing() && how != "" {
body := p.outsideBody(w)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+"."+how+"."+now.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the end of the wait of %s on %s is not said, and is said at the next pass: %v", w.Module, w.Node, err)
return
}
}
p.outsideFinding(w, false)
if ask, err := asks.OutsideAskKey(w.Node, w.Module, w.FP); err == nil {
p.remove(ctx, buckets.Asks(), ask)
}
if p.outside != nil && w.Cause != "" {
p.outside.mu.Lock()
p.outside.causes[w.Cause] = removeString(p.outside.causes[w.Cause], w.Node)
p.outside.mu.Unlock()
}
p.remove(ctx, buckets.Waits(), key)
}
func removeString(l []string, s string) []string {
var out []string
for _, x := range l {
if x != s {
out = append(out, x)
}
}
return out
}
// outsideFinding writes the finding delivery-waits-outside, or clears it.
func (p *passer) outsideFinding(w outside.Wait, raised bool) {
if p.findings.Put == nil {
return
}
o := assign.WaitsOutsideFinding(w.Node, w.Module, w.Cause, w.On, p.clock())
if raised {
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", "delivery."+o.ID, err)
}
return
}
if p.findings.Delete != nil {
if err := p.findings.Delete(mcseatKey(o)); err != nil {
p.logf("the finding %s is not cleared: %v", "delivery."+o.ID, err)
}
}
}
func mcseatKey(o assign.Observed) string {
return views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
}
// outsideAsk asks the operator at the wait's bound, once, and returns the asking's id.
func (p *passer) outsideAsk(ctx context.Context, node, module, fp string, w outside.Wait, now time.Time) string {
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return ""
}
asking := assign.AskingSeconds(now)
a := assign.OutsideAsk(key, asks.OutsideAbout(key), node, module, fp, w.On, w.ClockFrom, asking, now)
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
if ask, err := assign.AskOf(a); err == nil {
return ask.ID
}
return ""
}
// wakeAt makes a pass when t comes: the settle's end and the bound are the waits' own times, the one wake (no polling).
func (p *passer) wakeAt(t, now time.Time) {
if p.outside == nil || p.outside.wake == nil {
return
}
time.AfterFunc(max(t.Sub(now), 0)+time.Millisecond, p.outside.wake)
}
@@ -1,333 +0,0 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The whole of the wait on an outside cause in the pass (design 47 G §706): a fixture keeper answering `conditions` and saying
// its events, a failing check of nats on ace, and the events, finding, ask and record the pass leaves.
type outsideRig struct {
t *testing.T
p *passer
now time.Time
conds []map[string]any
unread bool
events []string
bodies []meshwants.Wait
found map[string]bool
woke chan struct{}
health string
}
func newOutsideRig(t *testing.T) *outsideRig {
r := &outsideRig{t: t, now: time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC), found: map[string]bool{}, woke: make(chan struct{}, 16), health: nodeengineclient.HealthFailed}
r.p = newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, r.now)
r.p.now = func() time.Time { return r.now }
r.p.w.put(buckets.Asks(), "x", nil)
r.p.outside = newOutsideActs(func(key string, body any) (json.RawMessage, error) {
if r.unread {
return nil, context.DeadlineExceeded
}
return json.Marshal(map[string]any{"conditions": r.conds})
}, func() { r.woke <- struct{}{} }, func(string, ...any) {}, 30*time.Minute, 90*time.Second)
r.p.publishWants = func(event string, body any, id string) error {
r.events = append(r.events, event)
r.bodies = append(r.bodies, body.(meshwants.Wait))
return nil
}
r.p.findings.Put = func(f mcseat.Finding) error { r.found[f.Scope+"."+f.ID+"."+f.Kind] = true; return nil }
r.p.findings.Delete = func(key string) error { delete(r.found, key); return nil }
return r
}
func (r *outsideRig) cond(key string, also ...string) {
r.conds = append(r.conds, map[string]any{"key": key, "subject": map[string]any{"machine": strings.Split(key, ".")[1], "also": also}})
}
// review is one pass's judging of nats on ace at the fingerprint, with the read answered at once when one is started.
func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) assign.Nodes {
r.t.Helper()
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
healthy := r.health == nodeengineclient.HealthOK
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
if moved == nil {
moved = func(string, string) bool { return false }
}
out := r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
select {
case <-r.woke: // the read was answered: the next pass judges by it
out = r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
case <-time.After(50 * time.Millisecond):
}
return out
}
func (r *outsideRig) wait() outside.Wait {
r.t.Helper()
for _, v := range r.p.w.read(buckets.Waits()) {
w, err := outside.Decode(v)
if err != nil {
r.t.Fatal(err)
}
return w
}
return outside.Wait{}
}
func TestTheWaitOnAnOutsideCause(t *testing.T) {
healthOf := func(n assign.Nodes) string { return n.Haves["ace"]["nats"].Health }
t.Run("a cause listing the machine is a wait said at once, ended cleared when it clears healthy", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 1 || r.events[0] != meshwants.EventWaitHeld {
t.Fatalf("health %s events %v", healthOf(n), r.events)
}
b := r.bodies[0]
if b.Step != meshwants.WaitStepOutside || b.On != "unreachable:hub" || b.Bound != "30m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(r.now) || b.Node != "ace" || b.Module != "nats" {
t.Errorf("wait-held %+v", b)
}
if !r.found["delivery.ace.nats.delivery-waits-outside"] {
t.Errorf("findings %v", r.found)
}
// the cause clears and the module runs healthy
r.conds, r.health, r.now = nil, nodeengineclient.HealthOK, r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.hub.unreachable","event":"cleared","change":"cleared","at":"2026-11-11T10:01:00Z","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
if len(r.events) != 2 || r.events[1] != meshwants.EventWaitEnded || r.bodies[1].How != outside.HowCleared || len(r.found) != 0 || len(r.p.w.read(buckets.Waits())) != 0 {
t.Errorf("events %v bodies %+v findings %v", r.events, r.bodies, r.found)
}
})
t.Run("a cause not listing the machine settles, then the failure is the delivery's", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "g14")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 0 || !r.wait().Settling {
t.Fatalf("settle: health %s events %v wait %+v", healthOf(n), r.events, r.wait())
}
r.now = r.now.Add(91 * time.Second)
if n := r.review(false, nil); healthOf(n) != nodeengineclient.HealthFailed {
t.Errorf("past the settle the check still counts as failing: %s", healthOf(n))
}
})
t.Run("a cause turning into another keeps the wait and its clock; a rewritten naming a moved module fails at once", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
r.conds = nil
r.cond("machine.hub.network", "ace")
r.now = r.now.Add(5 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.network","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
last := len(r.bodies) - 1
if len(r.events) != 2 || r.events[last] != meshwants.EventWaitHeld || r.bodies[last].On != "network:hub" || !r.bodies[last].ClockFrom.Equal(*r.bodies[0].ClockFrom) {
t.Errorf("events %v bodies %+v", r.events, r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.ace.nats.rewritten")
n := r2.review(false, func(owner, node string) bool { return owner == "nats" })
if healthOf(n) == nodeengineclient.HealthFailed || !r2.wait().Settling {
t.Errorf("a rewritten of the moved module is its failure: %s %+v", healthOf(n), r2.wait())
}
})
t.Run("the keeper silent waits on conditions-unread, and its own module is judged with no read", func(t *testing.T) {
r := newOutsideRig(t)
r.unread = true
r.review(false, nil)
r.now = r.now.Add(90 * time.Second) // the settle has run with no answer
r.review(false, nil)
if len(r.bodies) != 1 || r.bodies[0].On != outside.OnUnread {
t.Fatalf("bodies %+v", r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.hub.unreachable", "ace")
if n := r2.review(true, nil); healthOf(n) != nodeengineclient.HealthFailed || len(r2.events) != 0 {
t.Errorf("the keeper's own failing check waits: %s %v", healthOf(n), r2.events)
}
})
t.Run("past the bound the operator is asked; Go on counts it healthy and Count as failed holds it, each said ended", func(t *testing.T) {
for how, want := range map[string]string{assign.OptionGoOn: nodeengineclient.HealthOK, assign.OptionFailed: nodeengineclient.HealthFailed} {
r := newOutsideRig(t)
r.cond("machine.hub.silent", "ace")
r.review(false, nil)
r.now = r.now.Add(31 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-changed", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
key, _ := asks.OutsideAskKey("ace", "nats", fpTest)
raw := r.p.w.value(buckets.Asks(), key)
if raw == nil {
t.Fatalf("%s: no ask at %s", how, key)
}
_, ask, err := assign.DecodeAsk(key, raw)
if err != nil || ask.About != "ace.nats."+strings.TrimPrefix(key[strings.LastIndex(key, ".")+1:], "") || len(ask.Options) != 2 {
t.Fatalf("ask %+v %v", ask, err)
}
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: how, Warrant: ask.ID + "@2026-11-11T10:32:00Z", Words: "the operator"})
r.p.w.put(buckets.Answers, key, v)
n := r.review(false, nil)
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != how || healthOf(n) != want {
t.Errorf("%s: events %v last %+v health %s", how, r.events, r.bodies[len(r.bodies)-1], healthOf(n))
}
if r.p.w.value(buckets.Asks(), key) != nil || len(r.found) != 0 {
t.Errorf("%s: the ask or the finding stands after the end", how)
}
// a check that goes on failing is not waited on again
r.now = r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
before := len(r.events)
r.review(false, nil)
if len(r.events) != before {
t.Errorf("%s: a decided check is waited on again", how)
}
}
})
t.Run("a newer want ends the wait replaced, and a restart judges it again", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}}
r.p.outsideReview(context.Background(), "nats", "src2:"+strings.Repeat("c", 64), []string{"ace"}, n, false, func(string, string) bool { return false })
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != outside.HowReplaced {
t.Errorf("events %v", r.events)
}
})
t.Run("its bounds are read at the start from the files the manifest declares (A-112)", func(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
env := map[string]string{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": write("wait", "45\n"), "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": write("settle", "7")}
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 45*time.Minute || settle != 7*time.Second {
t.Errorf("bound %v settle %v err %v, want 45m and 7s", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_WAIT_FILE"], env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = filepath.Join(dir, "missing"), write("empty", "")
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 0 || settle != 0 {
t.Errorf("bound %v settle %v err %v, want both defaults", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = write("bad", "soon")
if _, _, err := outsideSettings(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") {
t.Errorf("err %v, want one naming the file", err)
}
})
}
func TestAMachineWhoseOwnNetworkIsOpenGetsNoWant(t *testing.T) {
m := assign.Module{Name: "nats", FP: fpTest, Built: true, Nodes: []string{"ace"}, MachineHeld: func(node string) bool { return node == "ace" }}
ok, rule, why := assign.Holds(m, "ace", assign.Nodes{Online: map[string]bool{"ace": true}})
if ok || rule == 0 || !strings.Contains(why, "machine.ace.network") {
t.Errorf("holds = %v %d %q", ok, rule, why)
}
o := newOutsideActs(nil, func() {}, func(string, ...any) {}, 0, 0)
o.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if !o.Held("ace") || o.Held("g14") {
t.Error("the own network condition's event does not hold the machine")
}
o.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.ace.network","subject":{"id":"ace","scope":"machine","machine":"ace"}}`))
if o.Held("ace") {
t.Error("the machine is held after its condition cleared")
}
}
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
for k, v := range r.p.w.read(buckets.Waits()) {
s.st.put(waitsState, k, v)
}
got, err := s.doWaits(map[string]any{"module": "nats"})
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
t.Errorf("wait = %+v", w)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("an unknown how is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on"}, "shanks.node-tools"); err == nil || len(acts) != 0 {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on", "why": "the hub is being rebuilt"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if len(acts) == 0 || !strings.Contains(acts[0], "hand-act") {
t.Errorf("the hand act is not recorded first: %v", acts)
}
keys, _ := s.st.keys(waitsState)
raw, _, _, _ := s.st.get(waitsState, keys[0])
if w, _ := outside.Decode(raw); w.Release != "go-on" || !strings.Contains(w.By, "the hub is being rebuilt") {
t.Errorf("wait = %+v", w)
}
// the process takes the release at its next pass: ended go-on, counted healthy
r.p.w.put(buckets.Waits(), keys[0], raw)
n := r.review(false, nil)
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
}
// A-246: a dependent's delivery wait is listed beside them, and released by its module and step delivery, its hand act first.
dw, _ := json.Marshal(deliveryWaiting{Wait: meshwants.Wait{Step: meshwants.WaitStepDelivery, Module: "app", Part: "db", On: "db@1.1.0", Machines: []string{"g14"}}})
s.st.put(buckets.DeliveryWaitsState, "app", dw)
got, err = s.doWaits(map[string]any{"node": "g14"})
if d := got.(map[string]any)["delivery"].([]deliveryWaiting); err != nil || len(d) != 1 || d[0].Module != "app" {
t.Errorf("the delivery waits listed = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"module": "app", "step": "build", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of another step is taken")
}
if _, err := s.doRelease(map[string]any{"module": "web", "step": "delivery", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of a module waiting for no delivery is taken")
}
before := len(acts)
if _, err := s.doRelease(map[string]any{"module": "app", "step": "delivery", "why": "db is fine as it runs"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
raw, _, _, _ = s.st.get(buckets.DeliveryWaitsState, "app")
var released deliveryWaiting
if json.Unmarshal(raw, &released) != nil || !released.Released || !strings.Contains(released.By, "db is fine") || len(acts) != before+1 {
t.Errorf("the delivery wait after its release = %+v, hand acts %v", released, acts[before:])
}
// A-250: assigner_delivered_again records its hand act, writes the ask and answers asked; assigner_waits lists it.
if _, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0"}, "op"); err == nil {
t.Error("an ask with no why is taken")
}
ans, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0", "why": "the catalog missed it"}, "op")
if err != nil || ans.(map[string]any)["asked"] != "db" || len(acts) != before+2 {
t.Fatalf("assigner_delivered_again = %+v %v", ans, err)
}
got, _ = s.doWaits(map[string]any{})
if a := got.(map[string]any)["delivered-again"].([]againAsk); len(a) != 1 || a[0].Version != "1.1.0" || a[0].By != "op" {
t.Errorf("the asks listed = %+v", a)
}
}
@@ -1,300 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The tools of the waits on an outside cause (design 47 G §706, §276's row *delivery on an outside cause*): `assigner_waits` reads
// each, the settle included, and `assigner_release` with `how` takes one. The tool writes the release into the wait's record and the
// assigner's process ends the wait at its next pass, saying wait-ended with that how.
// toolWaits is the tool that reads the outside waits.
const toolWaits = "assigner_waits"
// waitsState is the assigner's state of waits, as the runtime names a module's own state.
const waitsState = buckets.WaitsState
// waitSaid is one wait as assigner_waits answers it: its record and where it stands.
type waitSaid struct {
Key string `json:"key"`
outside.Wait
// Settling's end is in the record; Ends is when the bound runs from clock-from, empty for a settle.
BoundEnds string `json:"bound-ends,omitempty"`
}
// actingSaid is one wait for an acting part's new version as assigner_waits answers it (actingwaits.go).
type actingSaid struct {
Key string `json:"key"`
acting.Wait
BoundEnds string `json:"bound-ends,omitempty"`
}
// doWaits is the tool assigner_waits.
func (s *putTools) doWaits(args map[string]any) (any, error) {
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
sort.Strings(keys)
out := []any{}
for _, key := range keys {
raw, _, found, err := s.st.get(waitsState, key)
if err != nil || !found {
continue
}
if acting.Is(key) || acting.IsOrder(key) {
if a, err := acting.Decode(raw); err == nil && (arg(args, "node") == "" || a.Node == arg(args, "node")) && (arg(args, "module") == "" || a.Module == arg(args, "module")) {
said := actingSaid{Key: key, Wait: a}
if d, err := time.ParseDuration(a.Bound); err == nil && !a.Released {
said.BoundEnds = a.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
out = append(out, said)
}
continue
}
w, err := outside.Decode(raw)
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
continue
}
said := waitSaid{Key: key, Wait: w}
if w.Standing() {
if d, err := time.ParseDuration(w.Bound); err == nil {
said.BoundEnds = w.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
}
out = append(out, said)
}
// And each dependent's wait for its dependency's delivery (design 47 G §376, A-246), from the state delivery-waits.
delivery := []deliveryWaiting{}
dkeys, err := s.st.keys(buckets.DeliveryWaitsState)
if err != nil {
return nil, fmt.Errorf("the delivery waits cannot be listed: %w", err)
}
sort.Strings(dkeys)
for _, key := range dkeys {
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil || !found {
continue
}
var w deliveryWaiting
if json.Unmarshal(raw, &w) != nil || (arg(args, "module") != "" && w.Module != arg(args, "module")) ||
(arg(args, "node") != "" && !slices.Contains(w.Machines, arg(args, "node"))) {
continue
}
delivery = append(delivery, w)
}
// And each ask of assigner_delivered_again not yet holding, with why (A-250).
again := []againAsk{}
akeys, err := s.st.keys(buckets.DeliveredAgainState)
if err != nil {
return nil, fmt.Errorf("the asks to say a version delivered again cannot be listed: %w", err)
}
sort.Strings(akeys)
for _, key := range akeys {
raw, _, found, err := s.st.get(buckets.DeliveredAgainState, key)
var a againAsk
if err != nil || !found || json.Unmarshal(raw, &a) != nil || (arg(args, "module") != "" && a.Module != arg(args, "module")) {
continue
}
again = append(again, a)
}
return map[string]any{"waits": out, "delivery": delivery, "delivered-again": again}, nil
}
// toolDeliveredAgain is the tool that asks version-delivered said again for one version (design 47 G §376, A-250).
const toolDeliveredAgain = "assigner_delivered_again"
// doDeliveredAgain is assigner_delivered_again: the hand act is recorded first, then the ask is written into the state
// delivered-again, whose watch the assigner's process hears at once; it answers that it is asked.
func (s *putTools) doDeliveredAgain(args map[string]any, by string) (any, error) {
module, version, why := arg(args, "module"), arg(args, "version"), arg(args, "why")
if module == "" || version == "" || why == "" {
return nil, errors.New("assigner_delivered_again names the module, its version and why: it is kept in the hand-act log")
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_delivered_again %s %s", module, version), Why: why,
Cause: "version-delivered", Verb: "mesh-wants.assigner_delivered_again"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err)
}
data, err := json.Marshal(againAsk{Module: module, Version: version, By: by, Why: why})
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveredAgainState, keys.EscapeName(module), data); err != nil {
return nil, fmt.Errorf("the ask of %s %s is not written: %w", module, version, err)
}
return map[string]any{"asked": module, "version": version, "hand-act": taken.Log,
"follow": "the assigner says version-delivered again once every present machine placing it runs it healthy; assigner_waits says why it does not yet"}, nil
}
// deliveredAgainTool is assigner_delivered_again as the runtime serves it.
func (s *putTools) deliveredAgainTool() stdio.Tool {
return stdio.Tool{Name: toolDeliveredAgain, Description: "Say version-delivered again for one version of a module, for the operator and the healers " +
"(design 47 G §376): written in the hand-act log first, then asked of the assigner, which says it once every present machine placing " +
"the module runs that version healthy; until then assigner_waits lists the ask with why it does not hold. Answers that it is asked.",
Input: schema(map[string]string{"module": "the module", "version": "its MAJOR.MINOR.PATCH", "why": "why, kept in the hand-act log"},
[]string{"module", "version", "why"}),
Run: func(a map[string]any) (any, error) { return s.doDeliveredAgain(a, stdio.CallerOf(a)) }}
}
// doReleaseDelivery is assigner_release with step delivery: the hand act is recorded first, then the release is written on the
// module's delivery wait, which the assigner's process ends released at its next pass, writing the module's want.
func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error) {
module, why := arg(args, "module"), arg(args, "why")
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
key := keys.EscapeName(module)
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil {
return nil, fmt.Errorf("the delivery wait of %s is not read: %w", module, err)
}
var w deliveryWaiting
if !found || json.Unmarshal(raw, &w) != nil || w.Released {
return nil, fmt.Errorf("%s waits for no delivery: assigner_waits lists the delivery waits", module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s step delivery (waits on %s)", module, w.On), Why: why,
Cause: "delivery-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Released, w.By = true, by+": "+why
data, err := json.Marshal(w)
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveryWaitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s's delivery wait is not written: %w", module, err)
}
return map[string]any{"released": module, "step": meshwants.WaitStepDelivery, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
// module, which the assigner's process takes at its next pass.
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
if how == acting.HowNow {
return s.doReleaseActing(node, module, why, by)
}
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
}
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
prefix := node + "." + module + "."
var key string
var w outside.Wait
for _, k := range keys {
if !strings.HasPrefix(k, prefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if c, err := outside.Decode(raw); err == nil && c.Decided == "" {
key, w = k, c
}
}
if key == "" {
return nil, fmt.Errorf("%s holds no wait of %s on an outside cause: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, how), Why: why,
Cause: "outside-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Release, w.By, w.Why = how, by+": "+why, why
data, err := outside.Encode(w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseActing is assigner_release with how `now`: the entry of module on node moves with its acting parts' running versions. The
// hand act is recorded first; the release is written on each standing wait of the entry, and the process says wait-ended at its next pass.
func (s *putTools) doReleaseActing(node, module, why, by string) (any, error) {
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
type one struct {
key string
w acting.Wait
}
var held []one
prefix, orderPrefix := acting.Prefix+node+"."+module+".", acting.OrderPrefix+node+"."+module+"."
for _, k := range keys {
if !strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, orderPrefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if w, err := acting.Decode(raw); err == nil && !w.Released {
held = append(held, one{k, w})
}
}
if len(held) == 0 {
return nil, fmt.Errorf("%s holds no wait of %s for an acting part: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, acting.HowNow), Why: why,
Cause: "acting-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
released := []string{}
for _, h := range held {
h.w.Released, h.w.By, h.w.Why = true, by+": "+why, why
data, err := acting.Encode(h.w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, h.key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
released = append(released, h.key)
}
return map[string]any{"released": released, "how": acting.HowNow, "by": by, "hand-act": taken.Log}, nil
}
// waitsTool is assigner_waits as the runtime serves it.
func (s *putTools) waitsTool() stdio.Tool {
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: an entry waiting for an acting part's new version in composing or placing (§266, §268), with its part, version, step, clock-from and bound; and a moved module's failing health check on a " +
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
"machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil), Run: s.doWaits}
}
@@ -1,6 +0,0 @@
package main
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,539 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat
// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its
// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state`
// section `pins` (4 MiB), one key `trunk.<n>` per assignment graph revision whose value is the list of the records pinning
// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys:
// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision
// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names
// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up
// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being
// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state.
const (
pinsState = "pins"
pinsTool = "assigner_pins"
unpinTool = "assigner_unpin"
pinsMadeKey = "made"
pinsStatedKey = "pins-stated"
pinsCapBytes = 4 << 20
pinMissingKind = "pin-missing"
)
// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held
// when a record pinned it.
type PinsValue struct {
Records []string `json:"records"`
Missing bool `json:"missing,omitempty"`
}
// pinKey is the key of an assignment graph revision, its object's own name (A-169).
func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) }
// pinRevision is the revision a key of the state names, and whether it names one.
func pinRevision(key string) (int64, bool) {
n, ok := strings.CutPrefix(key, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// pinObjects is the object bucket of the revisions as the pins remove from it.
type pinObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// pinKeeper keeps the pins and removes what nothing pins.
type pinKeeper struct {
ctx context.Context
st endStore
objects pinObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
findings assign.Findings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// caught is whether the last event applied left none of the consumer's messages pending.
caught bool
}
// edit applies change to the value at key by compare-and-set, again while another write comes between.
func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error {
for {
raw, rev, err := k.st.get(key)
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %s do not read: %w", key, err)
}
}
next, write := change(v, raw != nil)
if !write {
return nil
}
value, _ := json.Marshal(next)
if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// held says the revision's object is kept.
func (k *pinKeeper) held(revision int64) (bool, error) {
_, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision))
if errors.Is(err, assigngraph.ErrNoObject) {
return false, nil
}
return err == nil, err
}
// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding
// pin-missing naming the record.
func (k *pinKeeper) pin(p mdseat.Pins) error {
for _, r := range p.Assignment {
held, err := k.held(r)
if err != nil {
return err
}
err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) {
if slices.Contains(v.Records, p.Record) && (held || v.Missing) {
return v, false
}
if !slices.Contains(v.Records, p.Record) {
v.Records = append(v.Records, p.Record)
slices.Sort(v.Records)
}
v.Missing = v.Missing || !held
return v, true
})
if err != nil {
return err
}
if !held {
k.missing(r, p.Record)
}
}
return nil
}
// free removes record from each assignment graph revision of p; a freed applied twice changes nothing.
func (k *pinKeeper) free(p mdseat.Pins) error {
for _, r := range p.Assignment {
err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) {
i := slices.Index(v.Records, p.Record)
if !found || i < 0 {
return v, false
}
v.Records = slices.Delete(v.Records, i, i+1)
return v, true
})
if err != nil {
return err
}
}
return nil
}
// missing writes the finding pin-missing, scope the assigner, id the revision.
func (k *pinKeeper) missing(revision int64, record string) {
if k.findings.Put == nil {
k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record)
return
}
o := assign.Observed{Scope: assignment.Module(), ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind,
Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision),
Headline: "A delivery record pins a revision the assigner no longer holds",
Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool),
Source: assignment.Module()}
if err := k.findings.Put(o.Finding(k.now())); err != nil {
k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err)
}
}
// made writes the key made once, create-only; one there already stays.
func (k *pinKeeper) made() error {
raw, _, err := k.st.get(pinsMadeKey)
if err != nil || raw != nil {
return err
}
value, _ := json.Marshal(k.now().UTC())
if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) {
return err
}
return nil
}
// timeAt is the time a key holds; false when it holds none.
func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) {
raw, _, err := k.st.get(key)
if err != nil || raw == nil {
return time.Time{}, false, err
}
var t time.Time
if err := json.Unmarshal(raw, &t); err != nil {
return t, false, fmt.Errorf("the key %s does not read: %w", key, err)
}
return t, true, nil
}
// stated keeps the at of a pins-stated applied.
func (k *pinKeeper) stated(at time.Time) error {
for {
raw, rev, err := k.st.get(pinsStatedKey)
if err != nil {
return err
}
var was time.Time
if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) {
return nil
}
value, _ := json.Marshal(at.UTC())
if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// statedSinceMade says a pins-stated said after the state was made was applied.
func (k *pinKeeper) statedSinceMade() (bool, error) {
made, ok, err := k.timeAt(pinsMadeKey)
if err != nil || !ok {
return false, err
}
at, ok, err := k.timeAt(pinsStatedKey)
if err != nil || !ok {
return false, err
}
return at.After(made), nil
}
// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied.
func (k *pinKeeper) hear(m jetstream.Msg) {
err := k.apply(m.Subject(), m.Data())
if err != nil {
k.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
caught := false
if meta, merr := m.Metadata(); merr == nil {
caught = meta.NumPending == 0
}
k.mu.Lock()
was := k.caught
k.caught = caught
k.mu.Unlock()
if caught && (!was || m.Subject() != mdseat.SubjectPinned) {
k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up
}
}
// apply takes one event's body.
func (k *pinKeeper) apply(subject string, data []byte) error {
switch subject {
case mdseat.SubjectPinned, mdseat.SubjectFreed:
var p mdseat.Pins
if err := json.Unmarshal(data, &p); err != nil || p.Record == "" {
k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err)
return nil
}
if subject == mdseat.SubjectPinned {
return k.pin(p)
}
return k.free(p)
case mdseat.SubjectPinsStated:
var s mdseat.PinsStatedEvent
if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() {
k.logf("%s holds no at, and is not taken: %v", subject, err)
return nil
}
return k.stated(s.At)
}
return nil // heads-replaced: a candidate's revision stops being current, judged after
}
// judge removes every revision neither pinned nor current, when the rule allows a removal at all.
func (k *pinKeeper) judge() {
k.mu.Lock()
caught := k.caught
k.mu.Unlock()
if !caught {
return
}
ok, err := k.statedSinceMade()
if err != nil {
k.logf("the pins are not judged: %v", err)
return
}
if !ok {
return // no pins-stated applied since the state was made: a pin said before may not be heard yet
}
if err := k.remove(); err != nil {
k.logf("the revisions nothing pins are not removed: %v", err)
}
}
// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted.
func (k *pinKeeper) remove() error {
names, err := k.objects.Names(k.ctx)
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := pinRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
raw, _, err := k.st.get(pinKey(r))
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %d do not read: %w", r, err)
}
}
if r == current || len(v.Records) > 0 {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
if raw != nil {
if err := k.st.drop(pinKey(r)); err != nil {
return fmt.Errorf("the pins of %d: %w", r, err)
}
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions no record pins and not current are removed", removed)
return nil
}
// hearPins registers the seat mesh-delivery's four events on the assigner's hub.
func hearPins(hub *process.EventHub, k *pinKeeper) error {
for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} {
if err := hub.Handle(s, k.hear); err != nil {
return fmt.Errorf("%s: %w", s, err)
}
}
return nil
}
// pinsTools are assigner_pins and assigner_unpin over the runtime's states.
func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " +
"the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " +
"was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.",
Run: func(map[string]any) (any, error) { return readPins(st) }},
{Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " +
"hand-act log first. The removal rule then applies.",
Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id",
"why": "why it is removed by hand"}, []string{"revision", "record", "why"}),
Run: func(a map[string]any) (any, error) {
record, _ := a["record"].(string)
why, _ := a["why"].(string)
return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a))
}}}
}
// readPins is what assigner_pins answers.
func readPins(st store) (any, error) {
ks, err := st.keys(pinsState)
if err != nil {
return nil, err
}
sort.Strings(ks)
revisions := map[string]PinsValue{}
out := map[string]any{"revisions": revisions}
var newest int64
bytes := 0
for _, key := range ks {
raw, _, found, err := st.get(pinsState, key)
if err != nil || !found {
continue
}
bytes += len(key) + len(raw)
if r, ok := pinRevision(key); ok {
var v PinsValue
if json.Unmarshal(raw, &v) == nil {
revisions[key] = v
}
newest = max(newest, r)
continue
}
out[key] = json.RawMessage(raw)
}
out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes}
if newest > 0 {
out["newest-pinned"] = newest
}
return out, nil
}
// unpin removes one record from one revision's list, the hand act recorded first.
func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) {
r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64)
switch {
case err != nil || r <= 0:
return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision)
case record == "" || why == "":
return nil, errors.New("name the record and say why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("an unpin names who asks it, and the runtime proved no caller")
}
key := pinKey(r)
raw, _, found, err := st.get(pinsState, key)
if err != nil {
return nil, err
}
var v PinsValue
if found {
if err := json.Unmarshal(raw, &v); err != nil {
return nil, err
}
}
i := slices.Index(v.Records, record)
if i < 0 {
return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool)
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record,
Why: why + " (" + by + ")", Cause: "unpin"})
if err != nil {
return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err)
}
v.Records = slices.Delete(v.Records, i, i+1)
value, _ := json.Marshal(v)
if err := st.put(pinsState, key, value); err != nil {
return nil, err
}
return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil
}
// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool
// process: the assigner's own state, heard as it changes, never on a timer.
func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) {
w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly())
if err != nil {
k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err)
return
}
go func() {
defer func() { _ = w.Stop() }()
for e := range w.Updates() {
if e == nil {
continue
}
if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut {
k.judge()
}
}
}()
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
func pinsBucket() string { return assignment.Module() + "_" + pinsState }
@@ -1,119 +0,0 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until
// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a
// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing,
// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays.
func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) {
objects := memObjects{}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
return d
}
shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
// A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept.
cand := entry("d", "fp-d2")
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
var found []mcseat.Finding
clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock },
logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" },
findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}}
if err := k.made(); err != nil {
t.Fatal(err)
}
say := func(subject string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if err := k.apply(subject, raw); err != nil {
t.Fatal(err)
}
k.judge()
}
pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} }
has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok }
k.caught = true
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r2", 1))
say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed
if !has(2) {
t.Fatal("revision 2 was removed before a pins-stated said after the state was made")
}
k.caught = false
say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)})
if !has(2) {
t.Fatal("revision 2 was removed while the consumer was behind")
}
k.caught = true
say(mdseat.SubjectFreed, pins("r1", 1))
if has(2) {
t.Error("revision 2, neither pinned nor current, is kept")
}
if !has(1) || !has(3) {
t.Error("revision 1 (pinned by r2) or 3 (current) was removed")
}
raw, _, _ := k.st.get(pinKey(1))
if string(raw) != `{"records":["r2"]}` {
t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw)
}
say(mdseat.SubjectFreed, pins("r2", 1))
if has(1) {
t.Error("revision 1 is kept once both records freed it")
}
if _, ok := objects[assigngraph.EntryObject(old)]; ok {
t.Error("the entry no kept revision lists and nothing names is kept")
}
if _, ok := objects[assigngraph.EntryObject(running)]; !ok {
t.Error("the entry of a running build is removed")
}
if _, ok := objects[assigngraph.EntryObject(shared)]; !ok {
t.Error("the entry the current revision lists is removed")
}
if _, ok := objects[assigngraph.EntryObject(cand)]; !ok {
t.Error("the entry a run's kept candidate lists is removed")
}
say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{})
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok {
t.Error("the gate's heads-replaced dropped a run's candidate")
}
say(mdseat.SubjectPinned, pins("r3", 2))
raw, _, _ = k.st.get(pinKey(2))
if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" {
t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found)
}
}
@@ -1,487 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as
// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and
// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written
// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing.
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
const (
writtenState = "written"
writtenTool = "assigner_written"
)
// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes.
const (
pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>"
)
// End is one value of the state written: the newest end of a module's pipeline on a watched branch.
type End struct {
Source int64 `json:"source"`
EndCount int64 `json:"end-count"`
Fp string `json:"fp"`
Commit string `json:"commit,omitempty"`
// Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed).
Passed bool `json:"passed"`
// Written: the wants were written from this end.
Written bool `json:"written,omitempty"`
Why string `json:"why,omitempty"`
}
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
func endKey(module, branch string) string {
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again.
func (e End) newer(source, count int64) bool {
return source > e.Source || (source == e.Source && count > e.EndCount)
}
// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed
// optional step holds nothing).
func requiredPassed(p mpr.PipelineEnded) bool {
for _, s := range p.Steps {
if s.Status != mpr.StepResultStatusRequired {
continue
}
switch s.Outcome {
case mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeCouldNotRun, mpr.StepResultOutcomeDeclined, mpr.StepResultOutcomeNotAsked,
mpr.StepResultOutcomeCancelled, mpr.StepResultOutcomeSuperseded, mpr.StepResultOutcomeDidNotBuild,
mpr.StepResultOutcomeRunning, mpr.StepResultOutcomeWaiting, mpr.StepResultOutcomeWaitingForABuildNode:
return false
}
}
return p.Outcome == mpr.PipelineEndedOutcomePassed
}
// endStore is the kept state written: the bus bucket, or a test's.
type endStore interface {
// get is a key's value and revision; none for a key not there.
get(key string) (value []byte, revision uint64, err error)
// swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not.
swap(key string, value []byte, revision uint64) error
// keys are the keys that stand, and drop deletes one.
keys() ([]string, error)
drop(key string) error
}
var errSwapped = errors.New("the key changed")
// kvEnds is endStore over a bus bucket.
type kvEnds struct {
ctx context.Context
kv jetstream.KeyValue
}
func (k kvEnds) get(key string) ([]byte, uint64, error) {
e, err := k.kv.Get(k.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (k kvEnds) swap(key string, value []byte, revision uint64) error {
var err error
if revision == 0 {
_, err = k.kv.Create(k.ctx, key, value)
} else {
_, err = k.kv.Update(k.ctx, key, value, revision)
}
if errors.Is(err, jetstream.ErrKeyExists) || (err != nil && isWrongSequence(err)) {
return errSwapped
}
return err
}
func (k kvEnds) keys() ([]string, error) {
ks, err := k.kv.Keys(k.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return ks, err
}
func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) }
func isWrongSequence(err error) bool {
var ae *jetstream.APIError
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
}
// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations
// (design 47 G §95, §353).
type pipelineEnds struct {
st endStore
wake func()
logf func(string, ...any)
// on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353).
on atomic.Bool
mu sync.Mutex
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
// 47 G §14).
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
trunk func() string
// runner is the module holding the seat mesh-pipeline-runner, as the last pass read it from the acting edges (A-334); nil
// until a pass read the graph.
runner atomic.Pointer[string]
// held are the group members' pipeline-ended heard before a pass read the graph, by their body: unacknowledged, so the
// bus keeps them, and taken once the runner is read (A-334), never dropped for the runner not being known yet.
held map[string]jetstream.Msg
}
// setRunner keeps the module holding the seat mesh-pipeline-runner, read by the pass from the acting edges (A-334); the
// first one takes the group members' ends held until the graph was read.
func (e *pipelineEnds) setRunner(module string) {
e.mu.Lock()
first := e.runner.Swap(&module) == nil
held := e.held
e.held = nil
e.mu.Unlock()
if !first {
return
}
for _, m := range held {
e.hearPipeline(m)
}
}
// holdUntilRead keeps m, a group member's pipeline-ended, until a pass reads the runner from the graph; false when the
// runner is read already, so m is taken now.
func (e *pipelineEnds) holdUntilRead(m jetstream.Msg) bool {
e.mu.Lock()
defer e.mu.Unlock()
if e.runner.Load() != nil {
return false
}
if e.held == nil {
e.held = map[string]jetstream.Msg{}
}
e.held[string(m.Data())] = m
e.logf("%s is a group member's end heard before the assignment graph is read: held until a pass reads which module is the runner (A-334)", m.Subject())
return true
}
// isRunner says whether module holds the seat mesh-pipeline-runner, as the last pass read it.
func (e *pipelineEnds) isRunner(module string) bool {
r := e.runner.Load()
return r != nil && *r != "" && *r == module
}
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
}
// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set.
func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error {
key := endKey(p.Module, p.Branch)
for {
raw, rev, err := e.st.get(key)
if err != nil {
return err
}
if raw != nil {
var kept End
if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) {
return nil // an older end, or this one again
}
}
end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fingerprint, Commit: p.Commit, Passed: passed, Why: p.Why}
value, err := json.Marshal(end)
if err != nil {
return err
}
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
return err
}
e.wake()
return nil
}
}
// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp.
func (e *pipelineEnds) markWritten(module, branch, fp string) {
key := endKey(module, branch)
for {
raw, rev, err := e.st.get(key)
var kept End
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written {
return
}
kept.Written = true
value, _ := json.Marshal(kept)
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
e.logf("the end of %s on %s is not marked written: %v", module, branch, err)
}
return
}
}
// passedAt says whether the newest end kept for module on branch passed at fp.
func (e *pipelineEnds) passedAt(module, branch, fp string) bool {
raw, _, err := e.st.get(endKey(module, branch))
var kept End
return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp
}
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
func (e *pipelineEnds) branchOf(module, placed string) string {
if placed != "" {
return placed
}
e.mu.Lock()
defer e.mu.Unlock()
if b := e.firsts[module]; b != "" {
return b
}
if e.trunk != nil {
return e.trunk()
}
return ""
}
// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, but the runner's: in a group
// take-in that changes the runner it is the acting part the other members wait on, so it goes first, wanted on its own end
// once that passed, ahead of the group (A-334, as the node-engine does, §268). A run on request delivers nothing (design 47 G
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)
case p.Run == "" && p.Group != "" && e.holdUntilRead(m):
return
case p.Run != "" || (p.Group != "" && !e.isRunner(p.Module)):
default:
if err := e.take(p, requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-<id>`) delivers
// nothing: it drops the run's candidate assignment graph (A-235).
func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
g, err := mprc.GroupEnded(m.Data())
if err != nil {
e.logf("%s is not taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
if strings.HasPrefix(g.Group, runGroupPrefix) {
if e.runEnded != nil {
if err := e.runEnded(g.Group); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
}
// hearEnds registers the two events on the assigner's hub.
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
}
if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil {
return fmt.Errorf("%s: %w", groupEndedSubject, err)
}
return nil
}
// writtenTools is the tool assigner_written: every key of the state written, with its end.
func writtenTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it.",
Run: func(map[string]any) (any, error) {
ks, err := st.keys(writtenState)
if err != nil {
return nil, err
}
sort.Strings(ks)
out := map[string]End{}
for _, k := range ks {
raw, _, ok, err := st.get(writtenState, k)
if err != nil || !ok {
continue
}
var e End
if json.Unmarshal(raw, &e) == nil {
out[k] = e
}
}
return out, nil
}}}
}
// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one.
func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string {
if !e.following() {
return nil
}
seen := map[string]bool{}
var out []string
for _, b := range placed[module] {
b = e.branchOf(module, b)
if !seen[b] {
seen[b] = true
out = append(out, b)
}
}
if len(out) == 0 {
out = append(out, e.branchOf(module, ""))
}
sort.Strings(out)
return out
}
// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already
// (a node placed later, a module taken away).
func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool {
if !e.following() {
return true
}
for _, w := range wanted {
if w.Fp == fp {
return true
}
}
for _, b := range branches {
if !e.passedAt(module, b, fp) {
return false
}
}
return true
}
// wrote marks the end the wants of module were written from.
func (e *pipelineEnds) wrote(module, fp string, branches []string) {
if !e.following() {
return
}
for _, b := range branches {
e.markWritten(module, b, fp)
}
}
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first
// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone
// is a registration removed: all its ends go.
func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
e.mu.Lock()
if gone || len(reg.Branches) == 0 {
delete(e.firsts, reg.Module)
} else {
e.firsts[reg.Module] = reg.Branches[0]
}
e.mu.Unlock()
kept, err := e.st.keys()
if err != nil {
e.logf("the ends kept cannot be listed: %v", err)
}
prefix := keys.EscapeName(reg.Module) + "."
for _, k := range kept {
rest, ok := strings.CutPrefix(k, prefix)
if !ok {
continue
}
branch, err := keys.UnescapeName(rest)
if err == nil && !gone && slices.Contains(reg.Branches, branch) {
continue
}
if err := e.st.drop(k); err != nil {
e.logf("the end %s is not deleted: %v", k, err)
}
}
if e.onBranches != nil && !gone {
e.onBranches(reg.Branches)
}
if !e.on.Swap(true) {
e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module)
}
e.wake()
}
// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb
// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own
// goroutine, the wants keep the trunk-only form until a registration is heard.
func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error {
regs := mcc.NewRegistrations(call)
regs.Follower.Logf = e.logf
regs.Follower.Changed = func(module string, value json.RawMessage, gone bool, _ time.Time) {
reg := mpcat.Registration{Module: module}
if !gone {
_ = json.Unmarshal(value, &reg)
}
e.registered(reg, gone)
}
if err := hub.Handle(mpcat.SubjectRegistrationChanged, func(m jetstream.Msg) {
if err := regs.Hear(ctx, m.Data()); err != nil {
e.logf("%s: %v", m.Subject(), err)
}
_ = m.Ack()
}); err != nil {
return err
}
go func() {
if err := regs.Start(ctx); err != nil && ctx.Err() == nil {
e.logf("the first view of the registrations is not answered: %v", err)
}
}()
return nil
}
@@ -1,401 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
mws "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// memEnds is endStore over a map, with revisions.
type memEnds struct {
vals map[string][]byte
revs map[string]uint64
}
func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil }
func (m *memEnds) keys() (out []string, _ error) {
for k := range m.vals {
out = append(out, k)
}
return out, nil
}
func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil }
func (m *memEnds) swap(key string, v []byte, rev uint64) error {
if m.revs[key] != rev {
return errSwapped
}
m.vals[key], m.revs[key] = v, rev+1
return nil
}
// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back.
type heardMsg struct {
jetstream.Msg
data []byte
subject string
acked, naked bool
}
func (m *heardMsg) Data() []byte { return m.data }
func (m *heardMsg) Subject() string { return m.subject }
func (m *heardMsg) Ack() error { m.acked = true; return nil }
func (m *heardMsg) Nak() error { m.naked = true; return nil }
func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded {
return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps,
Pipeline: branch + "." + module, Commit: "c"}
}
func step(status, outcome string) mpr.StepResult {
return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome}
}
// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch.
func TestAWantFollowsAPassedPipeline(t *testing.T) {
const fp1, fp2 = "fp1", "fp2"
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional
for _, c := range []struct {
name string
ends []mpr.PipelineEnded
group *mpr.GroupEnded
module string
branch string
fp string
may bool
}{
{"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false},
{"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true},
{"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false},
{"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false},
{"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true},
{"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true},
{"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true},
{"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false},
{"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true},
{"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false},
} {
woken := 0
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
e.on.Store(true)
for _, p := range c.ends {
if p.Group != "" {
continue // heard and ignored, as hearPipeline does
}
if err := e.take(p, requiredPassed(p)); err != nil {
t.Fatal(err)
}
}
if g := c.group; g != nil {
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
t.Fatal(err)
}
}
}
if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may {
t.Errorf("%s: may want = %v, want %v", c.name, got, c.may)
}
}
// A run on request's group-ended (group run-<id>, A-235) writes no want and drops the run's candidate; a drop that fails is
// heard again.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
var dropped []string
fail := false
e.runEnded = func(group string) error {
dropped = append(dropped, group)
if fail {
return fmt.Errorf("the bucket is not open")
}
return nil
}
raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed,
Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}})
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"}
e.hearGroup(m)
if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked {
t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked",
e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked)
}
fail = true
m = &heardMsg{data: raw, subject: m.subject}
e.hearGroup(m)
if !m.naked {
t.Error("a run's candidate not dropped is not heard again")
}
}
// A-334 (§29, §65, §268): in a group take-in that changes the runner, the runner's member (the holder of the seat
// mesh-pipeline-runner, read from the acting edges) is wanted on its own pipeline-ended once it passed, ahead of the group;
// the other members wait for group-ended.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
heard := func(p mpr.PipelineEnded) *heardMsg {
p.Group = "g"
raw, _ := json.Marshal(p)
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.x"}
e.hearPipeline(m)
return m
}
hear := func(p mpr.PipelineEnded) {
if m := heard(p); !m.acked {
t.Errorf("the pipeline-ended of %s is not acknowledged", p.Module)
}
}
// The runner's end heard before a pass read the graph is held, unacknowledged, and taken once the runner is read.
early := heard(pipe("runner", "main", 5, 1, "fp-runner", pass, step(req, "passed")))
if early.acked || early.naked || e.mayWant("runner", "fp-runner", []string{"main"}, nil) {
t.Fatalf("the runner's end before the graph is read: acked %v, naked %v, wanted %v; want it held", early.acked, early.naked,
e.mayWant("runner", "fp-runner", []string{"main"}, nil))
}
e.setRunner(assigngraph.Runner([]mws.AssignmentEntry{{Node: "ace", Module: "web", Edges: []mws.Edge{
{Kind: assigngraph.EdgeActing, Node: "ace", Module: "runner", Seat: mpr.Seat, Step: "pipeline"}}}}))
if !early.acked {
t.Error("the runner's end held until the graph was read is not taken once it is")
}
hear(pipe("web", "main", 5, 1, fp1, pass, step(req, "passed")))
if !e.mayWant("runner", "fp-runner", []string{"main"}, nil) || e.mayWant("web", fp1, []string{"main"}, nil) {
t.Errorf("a group changing the runner: may want the runner %v, web %v; want the runner ahead of the group and web not yet",
e.mayWant("runner", "fp-runner", []string{"main"}, nil), e.mayWant("web", fp1, []string{"main"}, nil))
}
hear(pipe("runner", "main", 6, 1, "fp-runner-2", fail, step(req, "failed")))
if e.mayWant("runner", "fp-runner-2", []string{"main"}, nil) {
t.Error("the runner's member that failed is wanted")
}
}
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) {
t.Error("a want naming the fingerprint was held")
}
// markWritten keeps the end and says it was written from; a newer end replaces it unwritten.
st := newMemEnds()
e = newPipelineEnds(st, func() {}, t.Logf)
e.on.Store(true)
_ = e.take(pipe("web", "main", 5, 1, fp1, pass), true)
e.wrote("web", fp1, []string{"main"})
var end End
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if !end.Written || end.Source != 5 {
t.Errorf("end = %+v, want written at source 5", end)
}
_ = e.take(pipe("web", "main", 6, 1, fp2, pass), true)
end = End{}
_ = json.Unmarshal(st.vals[endKey("web", "main")], &end)
if end.Written || end.Source != 6 {
t.Errorf("a newer end = %+v, want unwritten at source 6", end)
}
}
// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at
// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before.
func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
for _, c := range []struct {
name string
follow bool
passed bool
want bool
}{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
mdigest := strings.Repeat("e", 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", fp, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
digest := "sha256:" + strings.Repeat("c", 64)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
composes(t, p.w, "g14", digest)
placeOn(t, p.w, "g14", "web")
on := true
p.w.heardPresence("g14", !on, time.Now())
if c.follow {
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.on.Store(true)
if c.passed {
_ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true)
}
}
p.wants(context.Background())
key, _ := mwc.WantKey("g14", "web")
if _, got := wantsKV.values[key]; got != c.want {
t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want)
}
if c.follow && c.passed {
raw, _, _ := p.ends.st.get(endKey("web", trunkBranch))
var end End
_ = json.Unmarshal(raw, &end)
if !end.Written {
t.Errorf("%s: the end was not marked written", c.name)
}
}
}
}
// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and
// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default.
func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) {
st := newMemEnds()
woken := 0
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
e.trunk = func() string { return trunkBranch }
_ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false)
_ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true)
if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) {
t.Fatal("the follower held a want before any registration")
}
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
if !e.following() || woken == 0 {
t.Error("the first registration did not switch the wants on")
}
if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) {
t.Error("after the switch the wants did not follow the ends")
}
if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", ""))
}
e.registered(mpcat.Registration{Module: "web"}, true)
if len(st.vals) != 0 {
t.Errorf("a registration gone left %d ends", len(st.vals))
}
}
// Design 47 G §14, §184 (test 68): a node's entry is computed from the catalog half of the branch its placement names, so a
// change on `beta` moves only the want of the node whose placement chose `beta`.
func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
digest := "sha256:" + strings.Repeat("c", 64)
on := true
for _, node := range []string{"g14", "ace"} {
composes(t, p.w, node, digest)
p.w.heardPresence(node, !on, time.Now())
}
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
}
// publish makes the half of branch name web at fingerprint f: built, granted and passed by its pipeline.
publish := func(branch, f string, source int64) {
mdigest := strings.Repeat("e", 62) + fmt.Sprintf("%02d", source)
if branch == "beta" {
mdigest = strings.Repeat("f", 62) + fmt.Sprintf("%02d", source)
}
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, branch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
endDone(t, p.w, "web", f, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", f)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
_ = p.ends.take(pipe("web", branch, source, 1, f, mpr.PipelineEndedOutcomePassed), true)
}
run := func(node, f string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: f, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, node+".web", have)
}
fpWant := func(node string) string {
key, _ := mwc.WantKey(node, "web")
w, err := mwc.DecodeWant(wantsKV.values[key])
if err != nil {
t.Fatalf("the want of web on %s: %v", node, err)
}
return w.Fp
}
publish("main", fp("a"), 5)
publish("beta", fp("b"), 5)
run("g14", fp("a"))
run("ace", fp("b"))
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
}
publish("beta", fp("d"), 6)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
t.Errorf("a commit on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
}
}
// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node
// the rules read, and a registration's branches are each followed.
func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
w := newWorld()
woken := 0
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
entry := func(fp string) json.RawMessage {
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
return raw
}
h.changed("main", "web", entry("src2:1"), false)
h.changed("beta", "web", entry("src2:2"), false)
if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 {
t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken)
}
v := w.trunkOf("beta")["web"]
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
}
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
t.Errorf("graph node %+v", n)
}
h.changed("beta", "web", nil, true)
if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" {
t.Error("a module taken out of beta's half leaves main's")
}
}
@@ -1,95 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// placementBy is who a put the placement's build makes names as its asker: the placement, whose proposal the operator
// approved, is the warrant (design 47 *Going back is one act*; ADR 0666 §24).
const placementBy = "the placement"
// placementPuts makes each placed module's `build` a held put of that build, and releases a put the placement made once its
// placement names that build no more (design 47 line 912: "a return to an earlier build is one value in the placement, the
// module's `build` in its placement key, set by a placement proposal"; ADR 0666 §24; B-K1). The put then goes its own steps
// (the builder's build, the composition, an ask where the module's policy needs one, the want, holding) and is read with
// `assigner_puts` and released with `assigner_release` like any other; a placement's build the builder never built is refused
// by its build step, said in the put's words. A put asked by a person stands until the placement names another build.
func (p *passer) placementPuts(ctx context.Context) {
kv := p.kvs[buckets.Puts()]
if kv == nil {
return
}
now := p.clock().UTC()
named := map[string]string{} // put key -> the build its placement names
for node, a := range p.w.assignments() {
for _, m := range a.Modules {
if m.Build == "" {
continue
}
if key, err := puts.Key(node, m.Module); err == nil {
named[key] = m.Build
}
}
}
held := p.w.read(buckets.Puts())
keys := make([]string, 0, len(named)+len(held))
for k := range named {
keys = append(keys, k)
}
for k := range held {
if _, there := named[k]; !there {
keys = append(keys, k)
}
}
sort.Strings(keys)
for _, key := range keys {
build, names := named[key]
var before *puts.Put
if raw, ok := held[key]; ok {
if got, err := puts.Decode(raw); err == nil {
before = &got
}
}
var next puts.Put
switch {
case names && before != nil && before.FP == build:
continue // made already, or a person's put of the same build: it goes its own steps, ended or not
case names:
words := fmt.Sprintf("asked by %s: it names build %s of %s", placementBy, shortFP(build), key)
if before != nil && !puts.Ended(before.Step) {
words += fmt.Sprintf("; it replaces the put of %s, which stood at %s", shortFP(before.FP), before.Step)
}
next = puts.Put{FP: build, Hold: true, Why: "the placement names this build (ADR 0666 §24)", By: placementBy, At: now}.Said(now, puts.Asked, words)
case before != nil && before.By == placementBy && !puts.Ended(before.Step):
next = before.Said(now, puts.Released, fmt.Sprintf("released by %s: it names no build of its own any more; the assigner's rules write the want again", placementBy))
default:
continue
}
data, err := puts.Encode(next)
if err != nil {
p.logf("the placement's put %s is not written: %v", key, err)
continue
}
if before == nil {
_, err = kv.Create(ctx, key, data)
} else if e, gerr := kv.Get(ctx, key); gerr != nil {
err = gerr
} else {
_, err = kv.Update(ctx, key, data, e.Revision())
}
if err != nil {
p.logf("the placement's put %s could not be written, and is written at the next pass: %v", key, err)
continue
}
p.w.put(buckets.Puts(), key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
if p.acts != nil {
p.acts.say(key, next)
}
}
}
@@ -5,22 +5,14 @@ import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
@@ -40,12 +32,6 @@ const PutEvent = "put"
type caller interface {
Call(key string, body any) (json.RawMessage, error)
CallOn(node, key string, body any) (json.RawMessage, error)
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
CallKept(key string, body any) (json.RawMessage, error)
// CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668).
CallKeptOn(node, key string, body any) (json.RawMessage, error)
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
CallFile(key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
@@ -60,8 +46,6 @@ type putActs struct {
snapped map[string]string
snapNo map[string]string
warranted map[string]bool
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
rec meshhandactsclient.Recorder
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
@@ -69,32 +53,32 @@ type putActs struct {
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{rec: meshhandactsclient.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p puts.Put) string {
func identity(key string, p delivery.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]puts.Put {
out := map[string]map[string]puts.Put{}
for key, value := range p.w.read(buckets.Puts()) {
node, module, err := puts.ParseKey(key)
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
}
put, err := puts.Decode(value)
put, err := delivery.DecodePut(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]puts.Put{}
out[module] = map[string]delivery.Put{}
}
out[module][node] = put
}
@@ -106,14 +90,13 @@ func (p *passer) puts(ctx context.Context) {
if p.acts == nil {
return
}
p.placementPuts(ctx)
all := p.putsByModule()
defer func() { p.putWaits(ctx, p.putsByModule()) }()
if len(all) == 0 {
return
}
assigned := p.w.assigned()
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
@@ -123,34 +106,34 @@ func (p *passer) puts(ctx context.Context) {
for _, module := range modules {
var manifest catalogue.Manifest
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
if got := p.w.manifest(mod.Manifest); got.err == "" {
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
manifest = got.manifest
}
}
for node, put := range all[module] {
if puts.Ended(put.Step) {
if delivery.PutEnded(put.Step) {
continue
}
key, _ := puts.Key(node, module)
key, _ := delivery.PutKey(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) {
if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed {
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, puts.Refused, "the bus's want is not written: "+err.Error())}
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) &&
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
@@ -159,67 +142,68 @@ func (p *passer) puts(ctx context.Context) {
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted {
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !mwc.SameWant(normalWant(before), normalWant(want)) {
p.putWant(ctx, node, module, want)
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, meshdeliveryclient.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w mwc.Want) mwc.Want {
func normalWant(w delivery.Want) delivery.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].Fp2}
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
w.Build, w.BuildWords = e.Outcome, e.Error
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
if c, ok := chosen[module]; ok {
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
} else if policy == "" {
if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := asks.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey)); err == nil && a.Asking == assign.PutAsking(put) {
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
case p.answeredWith(akey, assign.OptionNo):
w.Answer = assign.OptionNo
default:
expires := assign.ExpiresAt(ask)
w.AskExpired = !expires.IsZero() && w.Now.After(expires)
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
}
}
}
}
if r, ok := p.w.runningOf(node, module); ok {
w.Running = &r
}
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
w.Have = &h
}
}
@@ -229,16 +213,17 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
var out *assign.ComposedDigest
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
// from their new owner and nothing else changes.
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := grantsstate.PartGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
@@ -248,17 +233,17 @@ func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, b
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == puts.Person && next.Put.Ask != "" && w.Composed != nil {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey))
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
}
}
}
@@ -267,29 +252,29 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
}
}
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
func (p *passer) performs(key string, act occ.Act) error {
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks(), key))
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
func (p *passer) performs(key string, act asks.Act) error {
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
for _, o := range ask.Options {
for _, o := range a.Ask.Options {
if o.ID == assign.OptionYes {
return occ.Performs(o, act)
return o.Performs(act)
}
}
return fmt.Errorf("the ask %s offers no yes", key)
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b puts.Put) bool {
func samePut(a, b delivery.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put) bool {
kv := p.kvs[buckets.Puts()]
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
if kv == nil {
return false
}
@@ -298,11 +283,11 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := puts.Decode(e.Value())
now, err := delivery.DecodePut(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := puts.Encode(next)
data, err := delivery.EncodePut(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
@@ -313,7 +298,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
}
return false
}
p.w.put(buckets.Puts(), key, data)
p.w.put(delivery.AssignerPutsBucket, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
@@ -397,9 +382,8 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri
}
}
// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and
// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's
// record of the teller's ask says who chose and how (ADR 0274).
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
// router's record of the teller's ask says who chose and how (ADR 0274).
func (a *putActs) warrant(id, ask string) {
a.mu.Lock()
if a.warranted[id] {
@@ -409,10 +393,10 @@ func (a *putActs) warrant(id, ask string) {
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
a.mu.Unlock()
a.wake()
}
@@ -438,8 +422,8 @@ type putSaid struct {
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p puts.Put) {
node, module, err := puts.ParseKey(key)
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
if err != nil || a.bus == nil {
return
}
@@ -1,116 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its
// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint
// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release,
// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none.
// putWaitBound is the bound of a put without hold's wait.
const putWaitBound = 30 * time.Minute
// Wait-ended's how of a put's wait.
const (
putWaitRan = "ran"
putWaitReleased = "released"
putWaitSuperseded = "superseded"
)
// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run.
func putWaits(p puts.Put) bool {
return !p.Hold && p.Step == puts.Wanted
}
// putWaitBody is the wait of the put p of module on node.
func putWaitBody(node, module string, p puts.Put) meshwants.Wait {
since := p.StandsSince().UTC()
return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module(), Node: node, Module: module, Fingerprint: p.FP,
Since: since, ClockFrom: &since, Bound: putWaitBound.String(),
Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)}
}
// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204,
// A-211): its identity and the put as it was then.
type putWaiting struct {
ID string `json:"id"`
Put puts.Put `json:"put"`
}
// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left.
func (p *passer) saidPutWaits() map[string]putWaiting {
out := map[string]putWaiting{}
for key, value := range p.w.read(buckets.PutWaits()) {
var w putWaiting
if err := json.Unmarshal(value, &w); err != nil || w.ID == "" {
p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits(), key, err)
continue
}
out[key] = w
}
return out
}
// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended
// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said;
// one not said, or not kept, is said at the next pass. Without the state no put's wait is said.
func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) {
if p.kvs[buckets.PutWaits()] == nil {
return
}
waiting := p.saidPutWaits()
for key, was := range waiting {
node, module, err := puts.ParseKey(key)
if err != nil {
continue
}
cur, there := all[module][node]
same := there && identity(key, cur) == was.ID
if same && putWaits(cur) {
continue
}
how := putWaitSuperseded
if same && cur.Step == puts.Done {
how = putWaitRan
} else if same && cur.Step == puts.Released {
how = putWaitReleased
}
body := putWaitBody(node, module, was.Put)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil {
p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
p.remove(ctx, buckets.PutWaits(), key)
delete(waiting, key)
}
for module, byNode := range all {
for node, put := range byNode {
key, err := puts.Key(node, module)
if err != nil || !putWaits(put) {
continue
}
id := identity(key, put)
if w, said := waiting[key]; said && w.ID == id {
continue
}
if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil {
p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
kept := putWaiting{ID: id, Put: put}
p.put(ctx, buckets.PutWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
}
}
}
@@ -1,92 +0,0 @@
package main
import (
"context"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as
// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is
// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said
// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one
// that ran while the assigner was stopped gets its wait-ended at the next start.
func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) {
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
kept := &memKV{values: map[string][]byte{}}
var events []string
var bodies []meshwants.Wait
start := func() *passer { // a process's start: a fresh world that hands what the state holds
p := newPasser(kvSet{buckets.PutWaits(): kept}, at)
for k, v := range kept.values {
p.w.put(buckets.PutWaits(), k, v)
}
p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {})
p.publishWants = func(event string, body any, id string) error {
events, bodies = append(events, event), append(bodies, body.(meshwants.Wait))
return nil
}
return p
}
ctx := context.Background()
asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked")
wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed")
held := wanted
held.Hold = true
p := start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil {
t.Fatalf("a put that waits said %v, kept %q", events, kept.values)
}
events, bodies = nil, nil
// At the next start the put still waits: it was said before, nothing is said of it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 0 {
t.Fatalf("a put waiting at the start was said again: %v", events)
}
// It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}})
if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut ||
kept.values["ace.nats"] != nil {
t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values)
}
events, bodies = nil, nil
// A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing.
again := wanted
again.At = at.Add(time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("a put's written want said %v", events)
}
b := bodies[0]
if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil ||
!b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" {
t.Errorf("the put's wait is %+v", b)
}
// Said once.
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 {
t.Errorf("a put's wait was said again: %v", events)
}
// Put again under its key: the first wait ends superseded, the new one is held.
newer := asked
newer.At = at.Add(2 * time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}})
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest {
t.Errorf("a put asked again said %v %+v", events, bodies)
}
// The answer of assigner_puts.
if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" {
t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound)
}
if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" {
t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound)
}
}
@@ -1,177 +0,0 @@
package main
import (
"context"
"strconv"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
var revertReasons = []string{"restore point missing", "restore point damaged"}
// revertReason is the reason of a have's words when they say a held revert, or "".
func revertReason(words string) string {
for _, r := range revertReasons {
if strings.HasPrefix(words, r) {
return r
}
}
return ""
}
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
// that holds a revert, which backups leaves to it.
func (p *passer) reverts(ctx context.Context) map[string]bool {
taken := map[string]bool{}
wanted := p.w.wantsOf()
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := nodeengineclient.DecodeHave(value)
if err != nil {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := asks.RevertAskKey(node, module, want.Fp)
if err != nil {
continue
}
reason := revertReason(have.Detail)
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
if have.Fp == want.Fp && p.w.has(buckets.Asks(), akey) {
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
}
continue
}
taken[node+"."+module] = true
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
if !ok {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
if there != nil {
now := p.clock()
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
continue
}
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
continue // not answered, or answered hold: the hold stands and nothing is written
}
words := assign.RevertAnswerWords(answer.Option, current.Asking)
if want.Revert != mwc.RevertLive || want.Words != words {
want.Revert, want.Words = mwc.RevertLive, words
p.putWant(ctx, node, module, want)
}
}
return taken
}
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
if p.graph == nil || p.entries == nil {
return "", "", false
}
run, heard := p.w.runningOf(node, module)
if !heard || run.Stopped || run.Fingerprint == "" {
return "", "", false
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
return "", "", false
}
if !reverted {
return "", "", false
}
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
if !known {
return "", "", false // asked: its answer wakes a pass
}
var names, forms []string
for _, c := range changes {
f := 0
if c.Applied != nil {
f = c.Applied.Form
}
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
}
if len(names) == 0 {
return "", "", false
}
return strings.Join(names, ", "), strings.Join(forms, ", "), true
}
// revertFinding writes the finding revert-held, or clears it, once per process each way.
func (p *passer) revertFinding(o assign.Observed, raised bool) {
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
said := key + ".raised"
if !raised {
said = key + ".cleared"
}
if p.said[said] {
return
}
var err error
switch {
case raised && p.findings.Put != nil:
err = p.findings.Put(o.Finding(p.clock()))
case !raised && p.findings.Delete != nil:
err = p.findings.Delete(key)
default:
return
}
if err != nil {
p.logf("the finding %s is not written: %v", key, err)
return
}
p.said[said] = true
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
}
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
if p.entries == nil {
return nil, false
}
running, rs := p.entries.Need(ctx, runFp)
applied, as := p.entries.Need(ctx, wantFp)
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
return nil, false
}
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
}
changes, err := assigngraph.FormChanges(running, applied)
if err != nil {
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
return nil, false
}
return changes, true
}
@@ -1,136 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, at)
p.logf = t.Logf
findings := map[string][]byte{}
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
objects := memObjects{}
for i, fp := range []string{fpOld, fpTest} {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
entry := func(fp string, form int) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
}
answered := make(chan struct{}, 4)
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
wkey, _ := mwc.WantKey("ace", "web")
want := mwc.Want{Fp: fpOld, Self: true}
raw, _ := mwc.EncodeWant(want)
p.w.put(buckets.Wants(), wkey, raw)
hkey, _ := nodeengineclient.HaveKey("ace", "web")
have := func(fp, phase, detail string) {
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
}
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
pass := func() { p.backups(ctx, p.reverts(ctx)) }
pass() // the entries are asked; nothing is asked of the operator until they answer
for range 2 {
select {
case <-answered:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
}
}
pass()
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
if !p.w.has(buckets.Asks(), rkey) || p.w.has(buckets.Asks(), bkey) {
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
}
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks())[rkey])
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
t.Fatalf("the revert ask = %+v, %v", ask, err)
}
fkey := "delivery.ace.web.revert-held"
if _, raised := findings[fkey]; !raised {
t.Fatalf("findings %v: want %s", findings, fkey)
}
answer := func(option string) {
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
p.w.put(buckets.Answers, rkey, v)
}
puts := wantsKV.puts
answer(assign.OptionRevertHold)
pass()
if wantsKV.puts != puts {
t.Fatal("an answer hold wrote a want")
}
answer(assign.OptionRevertLive)
pass()
got, err := mwc.DecodeWant(p.w.read(buckets.Wants())[wkey])
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
t.Fatalf("the want after live = %+v, %v", got, err)
}
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
pass()
if _, raised := findings[fkey]; raised {
t.Fatal("the finding stands while the reverted build runs")
}
// A newer want replaces the held revert: the ask goes and the finding with it.
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
pass()
if _, raised := findings[fkey]; !raised {
t.Fatal("the finding is not raised again for a hold again")
}
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
p.w.put(buckets.Wants(), wkey, raw)
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
}
}
@@ -1,88 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running.<node>.<module>` is
// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every
// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it.
// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one.
func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
if err := w.hearRunning(m.Subject(), m.Data()); err != nil {
logf("%v", err)
return
}
wake()
}, logf)
if err != nil {
nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err)
return
}
w.mu.Lock()
w.runningHeard = true
w.mu.Unlock()
nh.Heard(nodeengine.StreamRunning)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearRunning keeps one `running` event said on subject.
func (w *world) hearRunning(subject string, data []byte) error {
var r nodeengine.Running
if err := json.Unmarshal(data, &r); err != nil {
return fmt.Errorf("%s holds what is not a running: %w", subject, err)
}
if r.Node == "" || r.Module == "" {
return fmt.Errorf("%s names no node or no module, so it is kept for none", subject)
}
w.mu.Lock()
w.running[r.Node+"."+r.Module] = r
w.mu.Unlock()
return nil
}
// runningOf is the last `running` heard of module on node.
func (w *world) runningOf(node, module string) (nodeengine.Running, bool) {
w.mu.Lock()
defer w.mu.Unlock()
r, ok := w.running[node+"."+module]
return r, ok
}
// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the
// have).
func (w *world) runningNow() map[string]map[string]nodeengine.Running {
w.mu.Lock()
defer w.mu.Unlock()
if !w.runningHeard {
return nil
}
out := map[string]map[string]nodeengine.Running{}
for _, r := range w.running {
if out[r.Node] == nil {
out[r.Node] = map[string]nodeengine.Running{}
}
out[r.Node][r.Module] = r
}
return out
}
@@ -1,773 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"reflect"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mdcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/judge"
)
// The seat mesh-module-assigner's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
// proposal one step, from what the world holds. A proposal the verbs wrote is said (assignments-proposed); the nodes it
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
// candidate; their answers are judged (assignments-judged) and, on a passing verdict, the operator is asked to approve it;
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (assignments-changed),
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
// lays the proposal again; a decline, an expiry or a withdrawal ends it (assignments-withdrawn). Every event is said on the
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
type publisher func(event string, body any, id string) error
// seatPublisher publishes the seat's events on mesh.seat.mesh-module-assigner.event.<event>, the seat's own subject, which
// the shared events stream keeps and every consumer of `mesh-module-assigner.<event>` hears (the seat's family, 7 days),
// with the headers an emit carries.
func seatPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatdef.EventSubject(mma.Seat, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", assignment.Module())
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
return err
}
}
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
// the controller: assignments is the one source (ADR 0567 §8, issue 774).
type seatActs struct {
bus caller
wake func()
logf func(string, ...any)
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
rec meshhandactsclient.Recorder
}
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
return &seatActs{bus: bus, wake: wake, logf: logf, rec: meshhandactsclient.Recorder{Ask: bus.Call}}
}
// record records once, in the hand-act log, that assignments was written on the operator's warrant (§5): the router's
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
// the controller's while that seat has no holder (ADR 0565 §11, meshhandactsclient.Recorder).
func (a *seatActs) record(warrant string) {
if a == nil || warrant == "" {
return
}
ask, _, _ := strings.Cut(warrant, "@")
go func() {
if taken, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
a.logf("the assignments's warrant %s is not recorded in the hand-act log: %v", ask, err)
} else {
a.logf("the assignments's warrant %s is recorded in the log of %s", ask, taken.Log)
}
}()
}
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
func (p *passer) clearStaleCalls() {
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
return
}
keys, err := p.findings.Keys()
if err != nil {
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
return
}
p.callsCleared = true
for _, k := range keys {
rest, ok := strings.CutPrefix(k, "module.")
if !ok {
continue
}
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
if node, module, two := strings.Cut(pair, "."); two {
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
p.callsCleared = false
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
}
}
}
}
}
}
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
func (p *passer) sayWaits(unbound []string) {
c := assign.SeatWaitsCondition(unbound)
state := string(c.State) + c.Summary
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
return
}
if c.State == occ.StateCleared && p.saidWaits == "" {
return // nothing was raised by this process
}
if err := p.say(c); err != nil {
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
return
}
p.said[c.Key], p.saidWaits = true, state
}
// seatAssignments is every node's assignments and the revision of its node key, as the watch holds them: the revision a proposal is laid over (§2, research 072 row 37).
func (p *passer) seatAssignments() (map[string]mma.Assignments, map[string]uint64) {
return p.w.assignmentKeys.AllRevisions()
}
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
func (p *passer) candidatesOf(id string) (map[string]mma.CandidatesByProposalNodeValue, map[string]uint64, time.Time) {
out, revs := map[string]mma.CandidatesByProposalNodeValue{}, map[string]uint64{}
var newest time.Time
for key, value := range p.w.read(plc.CandidatesBucket()) {
pid, node, err := plc.ParseCandidateKey(key)
if err != nil || pid != id {
continue
}
c, err := plc.DecodeCandidate(value)
if err != nil {
continue
}
meta := p.w.metaOf(plc.CandidatesBucket(), key)
out[node], revs[node] = c, meta.revision
if meta.at.After(newest) {
newest = meta.at
}
}
return out, revs, newest
}
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
func (p *passer) answersTo(id string) map[string]*nodeengine.CandidateAnswer {
out := map[string]*nodeengine.CandidateAnswer{}
for key, value := range p.w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := nodeengineclient.DecodeHaveNode(value)
if err != nil {
continue
}
if a, ok := h.Candidates[id]; ok {
out[node] = &a
}
}
return out
}
// assignmentsSeat takes every open proposal one step further.
func (p *passer) assignmentsSeat(ctx context.Context) {
for _, b := range seatBuckets() {
if p.kvs[b] == nil {
return
}
}
if p.saidProposed == nil {
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
}
if !p.copyRetired(ctx) {
return // the old state is copied whole before the seat takes a step: a half-copied state would place nothing on a node
}
current, revs := p.seatAssignments()
ids := make([]string, 0)
raws := p.w.read(plc.ProposalsBucket())
for id := range raws {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
prop, err := plc.DecodeProposal(raws[id])
if err != nil || prop.ID != id {
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
continue
}
p.stepProposal(ctx, prop, p.w.metaOf(plc.ProposalsBucket(), id).revision, current, revs)
}
}
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
kv := p.kvs[plc.PulsesBucket()]
_ = kv.Delete(ctx, key)
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
if err != nil {
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
return
}
p.w.putEntry(plc.PulsesBucket(), key, []byte(`{}`), rev, p.clock())
}
// pulsing says whether a bound's key stands.
func (p *passer) pulsing(key string) bool { return p.w.has(plc.PulsesBucket(), key) }
// deletePulses deletes every bound of a proposal.
func (p *passer) deletePulses(ctx context.Context, id string) {
for key := range p.w.read(plc.PulsesBucket()) {
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
if err := p.kvs[plc.PulsesBucket()].Delete(ctx, key); err == nil {
p.w.put(plc.PulsesBucket(), key, nil)
}
}
}
}
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string) (mma.Proposal, bool) {
a, err := assign.AssignmentsAsk(akey, prop, brief, assign.AssignmentsAsking(prop, p.clock()), p.clock())
if err != nil {
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
placed, err := assign.AskOf(a)
if err != nil {
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
if pk, err := plc.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
}
prop.State, prop.Ask = mmac.ProposalAsked, placed.ID
return prop, true
}
// stepProposal takes one proposal one step further.
func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64) {
if !p.saidProposed[prop.ID] && prop.State == mmac.ProposalAnswering && prop.Verdict == nil {
p.sayEvent(mma.EventAssignmentsProposed, mma.AssignmentsEvent{Proposal: prop}, prop.ID)
p.saidProposed[prop.ID] = true
}
if prop.Withdrawn != "" {
p.endProposal(ctx, prop, prop.Withdrawn)
return
}
if prop.Seed || prop.HandBack {
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: assignments is the mesh's alone")
return
}
if prop.Warrant != nil {
p.addNode(ctx, prop, revs)
return
}
akey, _ := asks.AssignmentsAskKey(prop.ID)
again := prop.Again != nil && !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == mmac.ProposalAsked {
askPulse, _ := plc.PulseAskKey(prop.ID)
switch {
case p.answeredWith(akey, assign.OptionYes):
p.warranted(ctx, prop, rev, akey, current, revs)
return
case p.answeredWith(akey, assign.OptionNo):
p.endProposal(ctx, prop, "declined by the operator")
return
case !p.pulsing(askPulse) && !again:
p.endProposal(ctx, prop, "its ask expired unanswered")
return
}
}
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
return
}
cands, candRevs, written := p.candidatesOf(prop.ID)
if len(prop.Must) == 0 {
for _, node := range plc.Nodes(prop) {
if _, ok := cands[node]; !ok && !p.laidHere[prop.ID] {
// The verb writes the proposal alone: its candidates have one writer, this process (design 47 G A-317).
p.lay(ctx, prop, rev, current, revs, prop.Words, true)
return
}
}
p.mustAnswer(ctx, prop, rev, current, cands)
return
}
for _, node := range prop.Must {
if _, ok := cands[node]; !ok {
return // a candidate just written is not heard yet: its watch wakes the next pass
}
}
if !written.IsZero() && (prop.Written == nil || written.After(*prop.Written)) {
prop.Written = &written
}
pulsing := map[string]bool{}
for _, node := range prop.Must {
if k, err := plc.PulseAnswerKey(prop.ID, node); err == nil {
pulsing[node] = p.pulsing(k)
}
}
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
return
}
prop.Verdict = &v
switch v.State {
case mdcc.VerdictFail:
prop.State, prop.Ask = mmac.ProposalFailed, ""
case plc.VerdictWaits:
prop.State, prop.Ask = mmac.ProposalAnswering, ""
case mdcc.VerdictPass:
if prop.State != mmac.ProposalAsked || again {
var ok bool
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
return
}
if again {
prop.Words = "judged and asked again (again)"
}
}
}
if p.writeProposal(ctx, prop, rev) {
p.sayEvent(mma.EventAssignmentsJudged, mma.AssignmentsEvent{Proposal: prop},
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
}
}
// sameVerdict says two verdicts say the same, their time aside.
func sameVerdict(a, b mma.Verdict) bool {
a.At, b.At = time.Time{}, time.Time{}
return reflect.DeepEqual(a, b)
}
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
func moved(prop mma.Proposal, revs map[string]uint64) bool {
for node, base := range prop.Base {
if revs[node] != uint64(base) {
return true
}
}
return false
}
// intendedOf is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
func (p *passer) intendedOf(prop mma.Proposal, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) map[string]map[string]bool {
key := fmt.Sprint(prop.ID, prop.Base)
if got, ok := p.intended[key]; ok {
return got
}
got, _ := p.compute(prop, current, cands)
p.intended[key] = got
return got
}
// compute is what the proposal changes on each node by its assignments, and the nodes that answer it: every node with a
// assignments or a record (judge.Intended, ADR 0593 §8). The holder composes nothing: each node says what its compositions
// change in its answer.
func (p *passer) compute(prop mma.Proposal, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) (map[string]map[string]bool, []string) {
laid := map[string]*mma.Assignments{}
for _, node := range plc.Nodes(prop) {
if c, ok := cands[node]; ok && !c.Reached {
laid[node] = c.Assignments
}
}
nodes := map[string]bool{}
for n := range current {
nodes[n] = true
}
for n := range p.w.records() {
nodes[n] = true
}
return judge.Intended(current, laid, slices.Sorted(maps.Keys(nodes)))
}
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
// change, and writes them into the proposal.
func (p *passer) mustAnswer(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, cands map[string]mma.CandidatesByProposalNodeValue) {
for _, node := range plc.Nodes(prop) {
if _, ok := cands[node]; !ok {
return // the verbs write the candidates before the proposal: wait to hear them
}
}
intended, must := p.compute(prop, current, cands)
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
for _, node := range must {
if _, ok := cands[node]; ok {
continue
}
key, err := plc.CandidateKey(prop.ID, node)
if err != nil {
continue
}
c := mma.CandidatesByProposalNodeValue{Proposal: prop.ID, Reached: true}
if pl, ok := current[node]; ok {
c.Assignments = &pl
}
p.put(ctx, plc.CandidatesBucket(), key, func() ([]byte, error) { return plc.EncodeCandidate(c) })
}
for _, node := range must {
if k, err := plc.PulseAnswerKey(prop.ID, node); err == nil {
p.pulse(ctx, k, judge.AnswerBound)
}
}
prop.Must = must
p.writeProposal(ctx, prop, rev)
}
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string) {
p.lay(ctx, prop, rev, current, revs, why, false)
}
// lay lays a proposal over the keys as they stand now and writes its candidates, each with what it adds (A-316); first says it is the
// proposal's first laying, the verb having written the proposal alone (A-317).
func (p *passer) lay(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Assignments, revs map[string]uint64, why string, first bool) {
held, recorded := p.lays(ctx)
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
Current: current, Revisions: revs, Held: held, Recorded: recorded, SeatListRead: p.seats.Heard()})
if slices.ContainsFunc(findings, plc.WaitsForSeatList) {
// ADR 0636 §1: nothing is ended on a list not read; the reader's change makes a pass, which lays it again.
p.logf("the proposal %s is not laid again yet: %s", prop.ID, p.seats.Health())
return
}
if len(findings) > 0 {
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
return
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
if p.laidHere == nil {
p.laidHere = map[string]bool{}
}
p.laidHere[prop.ID] = true
for _, node := range plc.Nodes(again) {
key, _ := plc.CandidateKey(prop.ID, node)
c := p.candidateOf(prop.ID, node, laid[node], current)
p.put(ctx, plc.CandidatesBucket(), key, func() ([]byte, error) { return plc.EncodeCandidate(c) })
}
again.Written, again.Words = ptr(p.clock().UTC()), why+": laid again over the keys as they stand, to be judged and asked again"
if first {
again.Words = why
}
p.writeProposal(ctx, again, rev)
}
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
// nodes with a record.
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
held := map[string]bool{}
for name := range p.w.trunk() {
held[name] = true
}
records := p.w.records()
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
}
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
// proposal deleted, and assignments-changed said.
func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, akey string, current map[string]mma.Assignments, revs map[string]uint64) {
if err := p.performs(akey, assign.AssignmentsAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
return
}
answer, _ := p.w.answer(akey)
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
return
}
cands, _, _ := p.candidatesOf(prop.ID)
laid := map[string]*mma.Assignments{}
for _, node := range plc.Nodes(prop) {
laid[node] = cands[node].Assignments
}
writes, err := assign.Writes(prop, laid)
if err != nil {
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
return
}
var changes []plc.Change
writtenLaid := map[string]*mma.Assignments{}
after := maps.Clone(current)
afterRevs := maps.Clone(revs)
for _, w := range writes {
rev2, err := p.writeNode(ctx, w.Node, w.Value)
if err != nil {
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
// again over the keys as this write left them.
p.logf("assignments %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
p.followWrite(prop, answer, changes)
if rest, ok := assign.Rest(prop, writtenLaid); ok {
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
} else {
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
}
return
}
before := current[w.Node]
now := mma.Assignments{Node: w.Node}
if w.Value != nil {
now = *w.Value
after[w.Node] = *w.Value
} else {
delete(after, w.Node)
}
afterRevs[w.Node] = rev2
c := plc.ChangeOf(before, now)
c.Before, c.After = w.Base, rev2
changes = append(changes, c)
writtenLaid[w.Node] = w.Value
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
p.followWrite(prop, answer, changes)
}
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
// not record it again; assignments-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
func (p *passer) addNode(ctx context.Context, prop mma.Proposal, revs map[string]uint64) {
current, _ := p.seatAssignments()
if moved(prop, revs) {
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
return
}
laid, findings := plc.LayChanges(current, prop.Changes)
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
node := prop.Changes[0].Node
if prop.Changes[0].Op == mmac.OpRemoveNode {
p.removeNode(ctx, prop, current, findings)
return
}
value, ok := laid[node]
if len(findings) > 0 || !ok {
p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; "))
return
}
rev, err := p.writeNode(ctx, node, &value)
if err != nil {
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
return
}
c := plc.ChangeOf(mma.Assignments{Node: node}, value)
c.Before, c.After = uint64(prop.Base[node]), rev
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
w := prop.Warrant
p.sayChanged(prop, mdc.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
[]plc.Change{c})
}
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; assignments-changed carries the warrant.
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
func (p *passer) removeNode(ctx context.Context, prop mma.Proposal, current map[string]mma.Assignments, findings []string) {
node := prop.Changes[0].Node
before, ok := current[node]
if len(findings) > 0 || !ok {
p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; "))
return
}
rev, err := p.writeNode(ctx, node, nil)
if err != nil {
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
return
}
c := plc.ChangeOf(before, mma.Assignments{Node: node})
c.Before, c.After = uint64(prop.Base[node]), rev
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID)
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
w := prop.Warrant
p.sayChanged(prop, mdc.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
Words: fmt.Sprintf("the machine %s left assignments: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
[]plc.Change{c})
}
// followWrite follows the keys a warrant wrote: the warrant recorded and assignments-changed said. No verb of the
// controller is called: the assignments is the one source of where modules run (ADR 0567 §8, issue 774).
func (p *passer) followWrite(prop mma.Proposal, answer mdc.Answer, changes []plc.Change) {
if len(changes) == 0 {
return
}
p.seat.record(answer.Warrant)
p.sayChanged(prop, answer, changes)
}
func sortedNodes(m map[string]*mma.Assignments) []string {
out := make([]string, 0, len(m))
for n := range m {
out = append(out, n)
}
sort.Strings(out)
return out
}
// sayChanged says assignments-changed for the keys written.
func (p *passer) sayChanged(prop mma.Proposal, answer mdc.Answer, changes []plc.Change) {
if len(changes) == 0 {
return
}
at := p.clock().UTC()
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
at = t
}
}
plc.SortChanges(changes)
p.sayEvent(mma.EventAssignmentsChanged, mma.AssignmentsEvent{Proposal: prop, Nodes: changeBodies(changes),
Warrant: &mma.AssignmentsEventWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
}
// endProposal ends a proposal: its ask, its candidates and its key deleted, and assignments-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop mma.Proposal, why string) {
if akey, err := asks.AssignmentsAskKey(prop.ID); err == nil && p.w.has(buckets.Asks(), akey) {
if err := p.kvs[buckets.Asks()].Delete(ctx, akey); err == nil {
p.w.put(buckets.Asks(), akey, nil)
}
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
if err := p.kvs[plc.ProposalsBucket()].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
return
}
p.w.put(plc.ProposalsBucket(), prop.ID, nil)
p.logf("the proposal %s ends: %s", prop.ID, why)
p.sayEvent(mma.EventAssignmentsWithdrawn, mma.AssignmentsEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
}
// deleteCandidates deletes every candidate of a proposal.
func (p *passer) deleteCandidates(ctx context.Context, id string) {
for key := range p.w.read(plc.CandidatesBucket()) {
if pid, _, err := plc.ParseCandidateKey(key); err == nil && pid == id {
if err := p.kvs[plc.CandidatesBucket()].Delete(ctx, key); err == nil {
p.w.put(plc.CandidatesBucket(), key, nil)
}
}
}
}
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
func (p *passer) writeProposal(ctx context.Context, prop mma.Proposal, rev uint64) bool {
raw, err := plc.EncodeProposal(prop)
if err != nil {
p.logf("the proposal %s is not written: %v", prop.ID, err)
return false
}
kv := p.kvs[plc.ProposalsBucket()]
var at uint64
if rev == 0 {
at, err = kv.Create(ctx, prop.ID, raw)
} else {
at, err = kv.Update(ctx, prop.ID, raw, rev)
}
if err != nil {
if !errors.Is(err, context.Canceled) {
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
}
return false
}
p.w.putEntry(plc.ProposalsBucket(), prop.ID, raw, at, p.clock())
return true
}
// say says one of the seat's events, said in the journal when the bus refuses it.
func (p *passer) sayEvent(event string, body mma.AssignmentsEvent, id string) {
if p.publish == nil {
return
}
if err := p.publish(event, body, id); err != nil {
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
}
}
// writeNode writes one node's assignments after (nil: none) as its keys (mesh-sdk mmac.Keys.Writes; research
// 072 row 37): the module keys that change, each carrying the value it replaces, then the node key, the commit, then the
// module keys no longer placed. It answers the node key's revision after (0: deleted). A write that fails up to the
// commit ends it with nothing changed for any reader, the module keys written before it read at their previous values;
// a delete after the commit that fails leaves a module key the node key does not name, which places nothing and is
// deleted at the node's next write. Each key written is taken into the world at once, so the next write of the pass
// reads it.
func (p *passer) writeNode(ctx context.Context, node string, after *mma.Assignments) (uint64, error) {
writes, err := p.w.assignmentKeys.Writes(node, after)
if err != nil {
return 0, err
}
kv := p.kvs[plc.AssignmentsBucket()]
committed, rev := false, uint64(0)
for _, w := range writes {
got, err := writeKey(ctx, kv, w)
if err != nil {
if committed {
p.logf("assignments %s is not deleted after %s's write (%v): it places nothing, and goes at the node's next write", w.Key, node, err)
continue
}
return 0, fmt.Errorf("assignments %s at revision %d: %w", w.Key, w.Base, err)
}
p.w.putEntry(plc.AssignmentsBucket(), w.Key, w.Value, got, p.clock())
if w.Commit {
committed, rev = true, got
}
}
return rev, nil
}
// writeKey makes one write of an assignments key by compare-and-set and answers the key's revision after (0: deleted).
func writeKey(ctx context.Context, kv jetstream.KeyValue, w mmac.KeyWrite) (uint64, error) {
switch {
case w.Value == nil:
return 0, kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
case w.Base == 0:
return kv.Create(ctx, w.Key, w.Value)
default:
return kv.Update(ctx, w.Key, w.Value, w.Base)
}
}
func ptr[T any](v T) *T { return &v }
// changeBodies are the changes as the seat's event carries them.
func changeBodies(changes []plc.Change) []map[string]any {
out := make([]map[string]any, 0, len(changes))
for _, c := range changes {
out = append(out, c.Body())
}
return out
}
@@ -1,655 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// The seat mesh-module-assigner's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
type seatStore interface {
store
del(state, key string) error
history(state, key string) ([]stdio.StateRevision, error)
}
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
// `<module>.<state>` where the manifest declares them under `reads`.
const haveState = "node-engine.have"
// The indexer's states the verbs read, each asked of the indexer's first-view tool of that name, never read from its bus
// state (novox/hq ADR 0596; ADR 0615 §1, ADR 0621 §1, §2: the manifest invokes them, and the indexer answers them to the
// assigner alone of the delivery's parts).
const (
indexerModules = "modules"
indexerManifests = "manifests"
)
// indexerView is one of the indexer's states whole, asked of its first-view tool through ask (stdio.Ask, or a test's) page
// after page, each page continued after the one before's next. It is asked again at every call: the indexer's state is
// the truth, and the verbs keep no copy of it.
func indexerView(ask func(string, any) (json.RawMessage, error), state string) (map[string]json.RawMessage, error) {
tool := buckets.IndexerModule + "." + state
if ask == nil {
return nil, fmt.Errorf("%s cannot be asked here", tool)
}
page := views.AskVia(ask, tool)
out := map[string]json.RawMessage{}
for after := ""; ; {
a, err := page(context.Background(), after)
if err != nil {
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
}
maps.Copy(out, a.Values)
if a.Next == "" || a.Next == after {
return out, nil
}
after = a.Next
}
}
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
type seatTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
now func() time.Time
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatlist.Part
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
return &seatTools{st: st, ask: ask, now: time.Now}
}
// keyed is one assignments key as a verb answers it.
type keyed struct {
Assignments mma.Assignments `json:"assignments"`
Revision uint64 `json:"revision"`
}
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
keys, err := s.st.keys(mma.StateAssignments)
if err != nil {
return nil, nil, fmt.Errorf("assignments cannot be listed: %w", err)
}
sort.Strings(keys)
held := mmac.NewKeys()
var bad []string
for _, key := range keys {
if _, _, _, err := mmac.ParseKey(key); err != nil {
continue
}
raw, rev, found, err := s.st.get(mma.StateAssignments, key)
if err != nil {
return nil, nil, fmt.Errorf("assignments key %s cannot be read: %w", key, err)
}
if !found {
continue
}
if _, err := held.Put(key, raw, rev, false); err != nil {
bad = append(bad, err.Error())
}
}
out, revs := held.AllRevisions()
if len(bad) > 0 && len(out) == 0 {
return nil, nil, fmt.Errorf("assignments cannot be read: %s", strings.Join(bad, "; "))
}
return out, revs, nil
}
// lays is what a proposal is checked against: the modules the indexer holds, as its first view of modules answers them,
// and the nodes with a record.
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
mods, err := indexerView(s.ask, indexerModules)
if err != nil {
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
}
held := map[string]bool{}
for k := range mods {
if name, err := meshcatalogclient.ParseModulesKey(k); err == nil {
held[name] = true
}
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
}
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
}
// proposal reads one open proposal.
func (s *seatTools) proposal(id string) (mma.Proposal, error) {
if err := mmac.CheckProposalID(id); err != nil {
return mma.Proposal{}, err
}
raw, _, found, err := s.st.get(mma.StateProposals, id)
if err != nil {
return mma.Proposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
}
if !found {
return mma.Proposal{}, fmt.Errorf("no proposal %s is open: mesh-module-assigner.proposals lists those that are", id)
}
return plc.DecodeProposal(raw)
}
// write writes a proposal laid by assign.Propose, whose write the assigner's process takes as the event to lay it: it writes the
// candidates, each with what it adds (design 47 G A-317, A-316); the verb writes none, so no node-engine hears a candidate before
// its parts.
func (s *seatTools) write(p mma.Proposal) error {
p.Written = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
if err != nil {
return err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
}
return nil
}
// proposed is what propose and adopt answer.
type proposed struct {
Proposal mma.Proposal `json:"proposal"`
Follow string `json:"follow"`
}
func followProposal(id string) string {
return fmt.Sprintf("the nodes it touches answer it; mesh-module-assigner.proposals with id %q says where it stands, and the "+
"operator is asked to approve it once its verdict passes", id)
}
// changesOf reads propose's changes: a JSON array, given as one or as its text.
func changesOf(v any) ([]mma.ProposalChange, error) {
var raw []byte
switch c := v.(type) {
case string:
raw = []byte(c)
case nil:
return nil, errors.New("propose names its changes")
default:
var err error
if raw, err = json.Marshal(c); err != nil {
return nil, err
}
}
var out []mma.ProposalChange
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
}
return out, nil
}
// doPropose is the verb propose (§2).
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
changes, err := changesOf(args["changes"])
if err != nil {
return nil, err
}
for _, c := range changes {
if c.Op == mmac.OpSeed {
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
current, revs, err := s.assignments()
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
warrant, err := warrantOf(args["warrant"])
if err != nil {
return nil, err
}
enrollers := plc.EnrollersOf(current)
now := s.now()
p, _, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()})
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
findings = append(findings, refusal)
}
if len(findings) > 0 {
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
func warrantOf(v any) (*mma.Warrant, error) {
var raw []byte
switch w := v.(type) {
case nil:
return nil, nil
case string:
if strings.TrimSpace(w) == "" {
return nil, nil
}
raw = []byte(w)
default:
var err error
if raw, err = json.Marshal(w); err != nil {
return nil, err
}
}
var out mma.Warrant
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
}
return &out, nil
}
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: the holder of the seat
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
// `person.<module>`), else the caller as proven, which the refusal names.
func moduleOfCaller(by string, enrollers []string) string {
for _, module := range enrollers {
if (stdio.Callers{module}).Allows(by) {
return module
}
}
return by
}
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
// assignments-withdrawn carrying it whole.
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
why := arg(args, "why")
if why == "" {
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
}
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
raw, err := plc.EncodeProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
}
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
// when its verdict passes.
func (s *seatTools) doAgain(args map[string]any) (any, error) {
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Again = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// doPlacement is the verb placement (design 47 G A-62, §704): the module's placements as the assignments rows hold them, one
// per branch a row names, with the nodes that run the module from it. A row naming no branch is one placement with the branch
// empty, as the row holds it.
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
module := arg(args, "module")
if module == "" {
return nil, errors.New("placement takes the module")
}
all, _, err := s.assignments()
if err != nil {
return nil, err
}
byBranch := map[string][]string{}
for node, a := range all {
for _, m := range a.Modules {
if m.Module == module {
byBranch[m.Branch] = append(byBranch[m.Branch], node)
}
}
}
out := mma.PlacementAnswer{Placements: []mma.ModulePlacement{}}
for _, branch := range slices.Sorted(maps.Keys(byBranch)) {
out.Placements = append(out.Placements, mma.ModulePlacement{Branch: branch, Nodes: slices.Sorted(slices.Values(byBranch[branch]))})
}
return out, nil
}
// assignmentsAnswer is what the verb assignments answers.
type assignmentsAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Count int `json:"count"`
History int `json:"history-per-key"`
Changes []historyEntry `json:"changes,omitempty"`
Older string `json:"older,omitempty"`
}
// historyEntry is one change of a key in the state's revisions.
type historyEntry struct {
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change plc.Change `json:"change"`
Deleted bool `json:"deleted,omitempty"`
}
// doAssignments is the verb assignments (§10, §6): one node's key, one module's nodes, or every key; with history, the
// changes in order from the revisions the state keeps.
func (s *seatTools) doAssignments(args map[string]any) (any, error) {
if arg(args, "seats") == "true" {
return s.seats(arg(args, "node"))
}
all, revs, err := s.assignments()
if err != nil {
return nil, err
}
node, module := arg(args, "node"), arg(args, "module")
out := assignmentsAnswer{Keys: map[string]keyed{}, Count: len(all), History: plc.AssignmentsHistory}
for n, p := range all {
if node != "" && n != node {
continue
}
if module != "" && !strings.Contains(" "+strings.Join(mmac.Assigned(p), " ")+" ", " "+module+" ") {
continue
}
out.Keys[n] = keyed{Assignments: p, Revision: revs[n]}
out.Nodes = append(out.Nodes, n)
}
sort.Strings(out.Nodes)
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
return nil, fmt.Errorf("%s has no assignments key", node)
}
if arg(args, "history") != "true" {
return out, nil
}
nodes := out.Nodes
if node != "" {
nodes = []string{node}
} else if module == "" {
nodes = nil
for n := range all {
nodes = append(nodes, n)
}
sort.Strings(nodes)
}
for _, n := range nodes {
revisions, err := s.nodeHistory(n)
if err != nil {
return nil, err
}
var before sketch
for _, r := range revisions {
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
after := sketch{}
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
after = sketchOf(r.Key, r.Value)
} else {
e.Deleted = true
}
e.Change = after.changeFrom(n, before)
e.Change.Before, e.Change.After = 0, r.Revision
before = after
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
continue
}
out.Changes = append(out.Changes, e)
}
}
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its assignments-changed event on "+
"the seat's subject on the shared events stream, kept %d days", plc.AssignmentsHistory, plc.EventDays)
return out, nil
}
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
func (s *seatTools) doProposals(args map[string]any) (any, error) {
ids, err := s.st.keys(mma.StateProposals)
if err != nil {
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
}
cands, err := s.st.keys(mma.StateCandidates)
if err != nil {
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
raw, _, found, err := s.st.get(haveState, k)
if err != nil || !found {
continue
}
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]nodeengine.CandidateAnswer{}
}
answers[id][node] = a
}
}
sort.Strings(ids)
out := []openProposal{}
for _, id := range ids {
if want := arg(args, "id"); want != "" && want != id {
continue
}
p, err := s.proposal(id)
if err != nil {
return nil, err
}
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
for _, k := range cands {
pid, node, err := plc.ParseCandidateKey(k)
if err != nil || pid != id {
continue
}
if _, rev, found, err := s.st.get(mma.StateCandidates, k); err == nil && found {
o.Candidates[node] = rev
}
}
out = append(out, o)
}
if arg(args, "id") != "" && len(out) == 0 {
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
}
return map[string]any{"proposals": out}, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
// only from a tool of that name), with the seat's own schemas.
func (s *seatTools) tools() []stdio.Tool {
run := map[string]func(map[string]any) (any, error){
mma.VerbAssignments: s.doAssignments,
mma.VerbProposals: s.doProposals,
mma.VerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
mma.VerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
mma.VerbAgain: s.doAgain,
mma.VerbPlacement: s.doPlacement,
}
var out []stdio.Tool
for _, v := range mma.Verbs() {
out = append(out, stdio.Tool{Name: mma.Seat + "." + v, Run: run[v]})
}
return out
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
keys, err := s.st.keys(mma.StateSeatHolders)
if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
}
sort.Strings(keys)
out := []mma.SeatHolders{}
for _, key := range keys {
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
}
if !found {
continue
}
var sh mma.SeatHolders
if json.Unmarshal(raw, &sh) != nil {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
continue
}
out = append(out, sh)
}
return map[string]any{"seats": out, "holds": len(keys)}, nil
}
// keyedRevision is one revision of a node's assignments key, the key it is of beside it.
type keyedRevision struct {
stdio.StateRevision
Key string
}
// nodeHistory is the revisions of a node's assignments, in the order the bus wrote them: its node key's `<node>.node`,
// whose every revision is one write of the node (research 072 row 37: the node key is a write's commit, so its history is
// the node's assignments history).
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
nodeKey, err := mmac.NodeKey(node)
if err != nil {
return nil, err
}
revisions, err := s.st.history(mma.StateAssignments, nodeKey)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", nodeKey, err)
}
var out []keyedRevision
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: nodeKey})
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
return out, nil
}
// sketch is a revision of a node's assignments as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (plc.ChangeOf).
type sketch struct {
modules []mma.AssignmentsByNodeNodeValueModulesItem
nodeWide string
}
// sketchOf reads one revision of a node key; a value that does not read places nothing.
func sketchOf(key string, value []byte) sketch {
_, _, kind, err := mmac.ParseKey(key)
if err != nil || kind != mmac.NodeKind {
return sketch{}
}
n, err := mmac.DecodeNode(value)
if err != nil {
return sketch{}
}
raw, _ := json.Marshal([]any{n.Pins, n.Holds})
return sketch{modules: n.Modules, nodeWide: string(raw)}
}
// changeFrom is what changed from before to s on node, as plc.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) plc.Change {
c := plc.Change{Node: node}
was := map[string]string{}
for _, r := range before.modules {
was[r.Module] = r.Digest
}
now := map[string]bool{}
nodeWide := before.nodeWide != s.nodeWide && len(before.modules) > 0
for _, r := range s.modules {
now[r.Module] = true
d, ok := was[r.Module]
switch {
case !ok:
c.Added = append(c.Added, r.Module)
case nodeWide || d != r.Digest:
c.Changed = append(c.Changed, r.Module)
}
}
for _, r := range before.modules {
if !now[r.Module] {
c.Removed = append(c.Removed, r.Module)
}
}
sort.Strings(c.Added)
sort.Strings(c.Removed)
sort.Strings(c.Changed)
return c
}
@@ -1,945 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"os"
"reflect"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatlist"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
)
func (s memStore) del(state, key string) error {
delete(s[state].values, key)
return nil
}
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
v, ok := s[state].values[key]
if !ok {
return nil, nil
}
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
}
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; assignments and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := mma.Assignments{V: keys.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}, {Module: "mesh-enrolment"}},
Holds: []mma.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: "mesh-enrolment"}}}
st := memStore{}
for _, state := range []string{mma.StateAssignments, mma.StateProposals, mma.StateCandidates, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
}
// Issue 1966: the indexer's modules are asked of its first view, never read from its bus state (ADR 0596, ADR 0621).
indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {}}}
for _, m := range []string{"web", "shell", "mesh-enrolment"} {
indexer.views["mesh-delivery-indexer.modules"][m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
var asked []string
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if raw, ok, err := indexer.answer(key, body); ok {
return raw, err
}
return json.RawMessage(`{"ok":true}`), nil
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
setAssignments(st[mma.StateAssignments], anchor)
// placement (design 47 G A-62): a module's placements as the rows hold them, one per branch, with the nodes that run it from there.
before := maps.Clone(st[mma.StateAssignments].values)
setAssignments(st[mma.StateAssignments], mma.Assignments{V: keys.Version, Node: "laptop", Modules: []mma.PlacedModule{{Module: "web", Branch: "feat/x"}}})
if got, err := s.doPlacement(map[string]any{"module": "web"}); err != nil ||
!reflect.DeepEqual(got, mma.PlacementAnswer{Placements: []mma.ModulePlacement{{Branch: "", Nodes: []string{"anchor"}}, {Branch: "feat/x", Nodes: []string{"laptop"}}}}) {
t.Errorf("placement of web = %+v, %v", got, err)
}
if got, err := s.doPlacement(map[string]any{"module": "ghost"}); err != nil || len(got.(mma.PlacementAnswer).Placements) != 0 {
t.Errorf("placement of a module placed nowhere = %+v, %v", got, err)
}
st[mma.StateAssignments].values = before
// withdraw and again mark the proposal; the holder ends or judges it.
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
if err != nil {
t.Fatal(err)
}
seed := got.(proposed).Proposal
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
t.Fatal(err)
}
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
t.Fatal(err)
}
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again == nil || p.Again.IsZero() {
t.Errorf("marked = %+v, %v", p, err)
}
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
t.Error("an unknown proposal was withdrawn")
}
delete(st[mma.StateProposals].values, seed.ID) // the holder ended it
st[mma.StateCandidates].values = map[string][]byte{}
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
!strings.Contains(err.Error(), "ghost") || len(st[mma.StateProposals].values) != 0 {
t.Errorf("a refused proposal: %v", err)
}
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
t.Error("an unknown change was taken")
}
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
move := got.(proposed).Proposal
if move.By != "operator@laptop" || move.Base["anchor"] != int64(len(st[mma.StateAssignments].values["anchor.node"])) || move.Base["laptop"] != 0 || len(st[mma.StateCandidates].values) != 0 {
t.Fatalf("propose = %+v", move)
}
// assignments and proposals read every state the seat keeps (§10); history names the stream for what is older.
reads := map[string]bool{}
pa, err := s.doAssignments(map[string]any{"module": "web", "history": "true"})
if a := pa.(assignmentsAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, "shared events stream") {
t.Errorf("assignments of web = %+v, %v", pa, err)
} else {
reads[mma.StateAssignments], reads["events"] = true, true
}
if _, err := s.doAssignments(map[string]any{"node": "laptop"}); err == nil {
t.Error("a node with no key was answered")
}
ps, err := s.doProposals(map[string]any{"id": move.ID})
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 0 {
t.Errorf("proposals = %+v, %v", ps, err)
} else {
reads[mma.StateProposals], reads[mma.StateCandidates] = true, true
}
raw0, err := readManifest()
if err != nil {
t.Fatal(err)
}
for _, state := range raw0.State {
if slices.Contains([]string{mma.StateAssignments, mma.StateProposals, mma.StateCandidates}, state.Name) && !reads[state.Name] {
t.Errorf("no verb reads the seat's state %s", state.Name)
}
}
for _, stream := range raw0.Streams {
if strings.HasSuffix(stream.Name, "-findings") { // the part's own finding events, not a state of the seat
continue
}
if !reads["stream "+stream.Name] {
t.Errorf("no verb reads the seat's stream %s", stream.Name)
}
}
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand assignments back to.
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
t.Errorf("a hold: %v", err)
}
// 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with
// the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused.
addNode := func(by, warrant string) error {
_, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace",
"decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by)
return err
}
good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`
for _, c := range []struct{ by, warrant, want string }{
{"novox.mesh-grants", good, "mesh-enrolment's alone"},
{"person.mesh-enrolment", good, "mesh-enrolment's alone"},
{"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
{"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"},
{"novox.mesh-enrolment", "", "carries no warrant"},
} {
if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want)
}
}
if err := addNode("novox.mesh-enrolment", good); err != nil {
t.Errorf("an add-node from mesh-enrolment: %v", err)
}
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off assignments: from
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
// of a node running modules, or with another node's warrant, it is refused.
setAssignments(st[mma.StateAssignments], mma.Assignments{V: keys.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
removeNode := func(node, warrant string) (any, error) {
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
}
for _, c := range []struct{ node, warrant, want string }{
{"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"},
{"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
} {
if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want)
}
}
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
t.Errorf("a remove-node from mesh-enrolment: %v", err)
} else if rm := got.(proposed).Proposal; !slices.Equal(plc.Nodes(rm), []string{"joiner"}) || !slices.Equal(plc.RemovedNodes(rm), []string{"joiner"}) {
t.Errorf("the remove-node proposed %+v", rm)
}
// 1126/B8: the seat serves its six verbs; adopt and hand-back went with the controller (ADR 0590).
if tools := s.tools(); len(tools) != 6 || tools[2].Name != "mesh-module-assigner.propose" || tools[2].Run == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
// Issue 1966: propose checks its modules against the indexer's first view of modules, every page of it, asked again at
// each call; a first view the indexer refuses refuses the proposal, naming the tool, and writes nothing.
for _, c := range []struct {
name, module, refused, want string
}{
{"a module on the last page", "web", "", ""},
{"a module the indexer does not hold", "ghost", "", "ghost"},
{"the first view refused", "web", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.modules does not answer"},
} {
t.Run("1966: "+c.name, func(t *testing.T) {
indexer.refused, indexer.asks = c.refused, nil
maps.DeleteFunc(st[mma.StateAssignments].values, func(k string, _ []byte) bool { return strings.HasPrefix(k, "joiner.") }) // the remove-node's joiner
st[mma.StateProposals].values, st[mma.StateCandidates].values = map[string][]byte{}, map[string][]byte{}
_, err := s.doPropose(map[string]any{"why": "x", "changes": []any{map[string]any{"op": "place", "node": "laptop", "module": c.module}}}, "op")
switch {
case c.want == "" && err != nil:
t.Errorf("propose: %v", err)
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want) || len(st[mma.StateProposals].values) != 0):
t.Errorf("propose: %v, want a refusal saying %q and nothing written", err, c.want)
}
if pages := indexer.asks["mesh-delivery-indexer.modules"]; c.refused == "" && pages != 3 {
t.Errorf("the first view of modules was asked %d pages, want its 3", pages)
}
})
}
indexer.refused = ""
// Issue 1966: the replicated mesh-dns-resolver held on two nodes is proposed through the verb as the bundle serves it,
// over the seat list it reads (ADR 0636 §1): unread, the check says it waits for the list and writes nothing; read, the
// proposal is laid.
for _, c := range []struct {
name string
silent bool
want string
}{
{"a replicated seat on two nodes waits for an unread seat list", true, "the seat list is not read yet, so whether the seat mesh-dns-resolver may be held on anchor and laptop waits for it"},
{"a replicated seat on two nodes, the seat list read", false, ""},
} {
t.Run("1966: "+c.name, func(t *testing.T) {
st[mma.StateProposals].values, st[mma.StateCandidates].values = map[string][]byte{}, map[string][]byte{}
s.seatList = gateOver(t, c.silent)
_, err := s.doPropose(map[string]any{"why": "a second resolver", "changes": []any{
map[string]any{"op": "hold", "node": "anchor", "module": "web", "claim": "mesh-dns-resolver", "scope": "mesh"},
map[string]any{"op": "place", "node": "laptop", "module": "web"},
map[string]any{"op": "hold", "node": "laptop", "module": "web", "claim": "mesh-dns-resolver", "scope": "mesh"}}}, "op")
switch {
case c.want == "" && err != nil:
t.Errorf("propose: %v", err)
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want) || strings.Contains(err.Error(), "held once") || len(st[mma.StateProposals].values) != 0):
t.Errorf("propose: %v, want a refusal saying %q, never held once, and nothing written", err, c.want)
}
})
}
s.seatList = nil
t.Run("1966: the seat list's again tool answers an operator's session through the mesh MCP server", func(t *testing.T) {
// Such a session is proved as its machine's tool runner (issue 627); the healers too, no other module. The list names
// the seats, resolved from the assigner's own seat-holders (A-295, A-298): a module holding neither is refused.
held := map[string][]byte{"node-tool-runner.shanks": []byte(`{"holders":[{"module":"node-tools","node":"shanks"}]}`),
"mesh-healers": []byte(`{"holders":[{"module":"mesh-delivery-watchman","node":"ace"}]}`)}
holders := seatlist.HoldersIn(func() (map[string][]byte, error) { return held, nil })
for caller, refused := range map[string]bool{"person.jochen": false, "shanks.node-tools": false, "ace.mesh-delivery-watchman": false,
"ace.mesh-healers": true, "ace.messenger": true} {
if err := seatlist.RefusalWith("assigner_seat_list_again", seatListAgainCallers, caller, holders); (err != nil) != refused {
t.Errorf("%s: refused=%v, err %v", caller, refused, err)
}
}
})
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
putAssignments(w, mma.Assignments{V: keys.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
hn, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
hk, _ := nodeengineclient.HaveNodeKey(node)
w.putEntry(buckets.Have, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("ace: %+v %v", d, ok)
}
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
}
findings := map[string][]byte{}
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
p.unplacedFindings()
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
putAssignments(w, mma.Assignments{V: keys.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
p.unplacedFindings()
if len(findings) != 0 {
t.Errorf("placed again: findings %v", findings)
}
})
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from assignments, only the keys a change moves, and answered by assignments seats", func(t *testing.T) {
writes := 0
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
// Issue 1966: nothing is written while the seat list is unread (ADR 0636 §1), and every seat the list names is no
// seat outside it once read.
p := &passer{w: newWorld(), logf: t.Logf, kvs: kvSet{plc.SeatHoldersBucket(): kv}, seats: gateOver(t, true)}
place := func(node string, holds ...mma.Holding) {
putAssignments(p.w, mma.Assignments{V: keys.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
}
place("ace", mma.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
place("g14", mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
ctx := context.Background()
if p.seatHolders(ctx); writes != 0 {
t.Fatalf("seat-holders was written from an unread seat list: %v", kv.values)
}
p.seats = gateOver(t, false)
p.seatHolders(ctx)
var broker mma.SeatHolders
if err := json.Unmarshal(kv.values["mesh-broker"], &broker); err != nil || broker.Outside {
t.Errorf("mesh-broker's holders %+v, %v: a seat the list names is no seat outside it", broker, err)
}
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
t.Fatalf("%d writes: %v", writes, kv.values)
}
p.seatHolders(ctx)
if writes != 3 {
t.Errorf("an unchanged assignments wrote %d keys", writes-3)
}
place("g14") // g14 holds nothing now
p.seatHolders(ctx)
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
}
st := memStore{mma.StateSeatHolders: &memKV{values: kv.values}}
out, err := newSeatTools(st, nil).seats("ace")
if err != nil || len(out.(map[string]any)["seats"].([]mma.SeatHolders)) != 2 {
t.Errorf("seats on ace = %+v, %v", out, err)
}
})
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
p := &passer{w: newWorld(), kvs: kvSet{plc.AssignmentsBucket(): kv}, now: time.Now, logf: t.Logf}
first := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}
if _, err := p.writeNode(ctx, "anchor", &first); err != nil || strings.Join(kv.log, " ") != "create anchor.web create anchor.node" {
t.Fatalf("the first write = %v, %v", kv.log, err)
}
raw := kv.values["anchor.node"]
// A write: the module keys, the node key (its commit), then the module key no longer placed.
after := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell", Ports: map[string]int64{"22": 2222}}}}
kv.log = nil
rev, err := p.writeNode(ctx, "anchor", &after)
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node delete anchor.web" || rev != kv.revs["anchor.node"] {
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
}
if got, _ := p.seatAssignments(); !mmac.SameAssignments(got["anchor"], after) {
t.Errorf("the write is not read back at once: %+v", got)
}
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
kv.refuse, kv.log = "anchor.node", nil
moved := mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell", Ports: map[string]int64{"22": 2223}}}}
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
t.Fatalf("a refused commit = %v, %v", kv.log, err)
}
fresh, err := mmac.AssignmentsOf(kv.values)
if err != nil || !mmac.SameAssignments(fresh["anchor"], after) {
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
}
kv.refuse = ""
// The verb's history reads the node key.
st := memStore{mma.StateAssignments: &memKV{values: kv.values}}
hs := newSeatTools(st, nil)
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
"anchor.node": {{Key: "anchor.node", Value: raw, Revision: 1, Op: "put"}, {Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
}}
pa, err := hs.doAssignments(map[string]any{"node": "anchor", "history": "true"})
a, _ := pa.(assignmentsAnswer)
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
t.Errorf("history = %+v, %v", pa, err)
}
// No assignments: the node key, then the module keys.
kv.log = nil
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete anchor.node delete anchor.shell" {
t.Errorf("the delete = %v, %v", kv.log, err)
}
})
t.Run("ADR 0641: the retired state is copied into assignments, key by key, with no gap and nothing deleted", func(t *testing.T) {
ctx := context.Background()
web, _ := mmac.AssignmentsValues(mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}})
shell, _ := mmac.AssignmentsValues(mma.Assignments{Node: "anchor", Modules: []mma.PlacedModule{{Module: "shell"}}})
oldWeb := func() map[string][]byte {
return map[string][]byte{"anchor.node": web["anchor.node"], "anchor.web": web["anchor.web"]}
}
withLegacy := oldWeb()
withLegacy["placement.anchor"] = []byte(`{"whole":1}`)
withOdd := oldWeb()
withOdd["a.b.c"] = []byte("x")
for _, c := range []struct {
name string
old map[string][]byte // the retired state
marked bool // the copy ended at an earlier start
present map[string][]byte // what assignments holds at the start
wantLog string // the writes into assignments, in order
wantValues map[string][]byte // assignments afterwards
journal int // lines said
said copiedSaid // the event; none when zero
}{
{"an empty new state takes every key, the node key last, and the old form is not copied", withLegacy, false, nil,
"create anchor.web create anchor.node", web, 1, copiedSaid{Copied: 2, Dropped: 1}},
{"a copy cut short by a crash is completed at the next start, and the marker is written", oldWeb(), false,
map[string][]byte{"anchor.web": web["anchor.web"]},
"create anchor.node", web, 1, copiedSaid{Copied: 1, Present: 1}},
{"a start after the marker copies nothing, even with a key missing", oldWeb(), true,
map[string][]byte{"anchor.node": shell["anchor.node"], "anchor.shell": shell["anchor.shell"]},
"", shell, 0, copiedSaid{}},
{"a key that fits no form is left and counted", withOdd, false, nil,
"create anchor.web create anchor.node", web, 1, copiedSaid{Copied: 2, Skipped: 1}},
{"an empty old state copies nothing and says nothing", nil, false, nil, "", nil, 0, copiedSaid{}},
} {
t.Run(c.name, func(t *testing.T) {
next := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
for k, v := range c.present {
next.write(k, v)
}
old := &retiredKV{values: map[string][]byte{}}
maps.Copy(old.values, c.old)
copies := &copiesKV{values: map[string][]byte{}}
if c.marked {
copies.values[copiedMarker] = []byte(`{}`)
}
var logs []string
bus := &emitted{}
p := &passer{w: newWorld(), kvs: kvSet{retiredBucket(): old, plc.AssignmentsBucket(): next, copiesBucket(): copies}, now: time.Now, seat: &seatActs{bus: bus},
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
if !p.copyRetired(ctx) || !p.copied {
t.Fatalf("the copy did not end: %v", logs)
}
if got := strings.Join(next.log, " "); got != c.wantLog {
t.Errorf("writes = %q, want %q", got, c.wantLog)
}
if c.wantValues != nil && !maps.EqualFunc(next.values, c.wantValues, slices.Equal[[]byte]) {
t.Errorf("assignments = %v, want %v", next.values, c.wantValues)
}
if _, ok := copies.values[copiedMarker]; !ok {
t.Errorf("the end of the copy is not marked in %s: %v", copiesBucket(), copies.values)
}
if !maps.EqualFunc(old.values, c.old, slices.Equal[[]byte]) || old.deletes != 0 {
t.Errorf("the retired state changed: %v, %d deletes; the readers delivered before still read it", old.values, old.deletes)
}
if len(logs) != c.journal {
t.Errorf("said %d lines %v, want %d", len(logs), logs, c.journal)
}
if c.said == (copiedSaid{}) {
if len(bus.events) != 0 {
t.Errorf("no copy was made, and said %v", bus.events)
}
} else if len(bus.events) != 1 || bus.events[0] != c.said {
t.Errorf("said %v, want the event %+v once", bus.events, c.said)
}
// A restart finds the marker: nothing is written, and no second event is said.
before := len(next.log)
p.copied = false
if !p.copyRetired(ctx) || len(next.log) != before || len(bus.events) > 1 {
t.Errorf("a second copy wrote %v, said %v", next.log[before:], bus.events)
}
if got := len(p.w.read(plc.AssignmentsBucket())); got != c.said.Copied {
t.Errorf("%d copied keys are in the world, want %d", got, c.said.Copied)
}
})
}
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{
Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"})
p.w.putEntry(buckets.Provisions, "ace.photos.db", failed, 1, time.Now())
p.kvs[buckets.Provisions] = &memKV{}
of := p.provisionsOf()
if got := of("photos")("ace"); got["ace.photos.db"].State != meshprovisionerclient.Failed {
t.Fatalf("provisions = %+v", got)
}
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Provided})
p.w.putEntry(buckets.Provisions, "ace.photos.db", provided, 2, time.Now())
p.provisionsOf()
if len(findings) != 0 {
t.Errorf("findings after provided = %v", findings)
}
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
ctx := context.Background()
p.kvs[plc.PulsesBucket()] = &memKV{values: map[string][]byte{}}
gone := map[string]bool{"ace.photos.s3": true}
p.provisionsMissing(ctx, gone)
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
}
p.provisionsMissing(ctx, gone)
if len(findings) != 0 {
t.Fatal("said before its bound")
}
p.w.put(plc.PulsesBucket(), "provision-missing.ace.photos.s3", nil) // the server expired it
p.provisionsMissing(ctx, gone)
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
t.Fatalf("past its bound: findings %v", findings)
}
p.provisionsMissing(ctx, map[string]bool{})
if len(findings) != 0 {
t.Errorf("keyed again: findings %v", findings)
}
})
}
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
func wantsVerbs(t *testing.T) {
fpOld, fpNew := meshcatalogclient.SourcePrefix2+strings.Repeat("1", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{policiesState, haveState, putsState, "findings", buckets.ComposedRead} {
st[state] = &memKV{values: map[string][]byte{}}
}
// Issue 1966: the indexer's modules and manifests are asked of its first views, never read from its bus state.
indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {}, "mesh-delivery-indexer.manifests": {}}}
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
// a fingerprint, and laptop's web refused there.
partNew, _ := catalogkey.KeyPart(fpNew)
webKey, _ := nodeengineclient.ComposedKey("anchor", "web", partNew)
refusedKey, _ := nodeengineclient.ComposedKey("laptop", "web", partNew)
st[buckets.ComposedRead].values["anchor.node"], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor"})
st[buckets.ComposedRead].values[webKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor", Module: "web", FP: fpNew})
st[buckets.ComposedRead].values[refusedKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := meshcatalogclient.ManifestDigest([]byte(raw))
indexer.views["mesh-delivery-indexer.manifests"][digest], _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: name, Manifest: raw})
indexer.views["mesh-delivery-indexer.modules"][name], _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{V: keys.Version, Fp2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
}
st[haveState].values["anchor.node"], _ = nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: "sha256:" + strings.Repeat("c", 64)})
var asked []string
var said []string
ask := func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if raw, ok, err := indexer.answer(key, body); ok {
return raw, err
}
return json.RawMessage(`{"ok":true}`), nil
}
w := newWantsTools(st, ask, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
policyOf := func(module string) policyRow {
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
if err != nil {
t.Fatal(err)
}
return out.(map[string]any)["modules"].([]policyRow)[0]
}
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
t.Errorf("photos = %+v", p)
}
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { // the policy its manifest says (issue 774)
t.Errorf("nats = %+v", p)
}
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
// controller being gone (1126/B8).
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
t.Errorf("compare = %+v", got)
}
pub, _ := out.(map[string]any)["published"].(map[string][]publishedRow)
if a, l := pub["anchor"], pub["laptop"]; len(a) != 2 || a[0].Module != "node" || a[1].Module != "web" || a[1].FP != fpNew ||
!strings.HasPrefix(a[1].Digest, "sha256:") || len(l) != 1 || l[0].Refused != "a required value is unset" || l[0].Digest != "" {
t.Errorf("published = %+v (unread %v)", pub, out.(map[string]any)["published-unread"])
}
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
// large states (the modules, their manifests, the composed record).
watches := map[string]int{}
byWatch := newWantsTools(st, ask, nil)
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
watches[state]++
for k, v := range st[state].values {
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true})
}
return nil
}, func(state, key string) ([]byte, bool, error) {
watches[state+" by key"]++
v, ok := st[state].values[key]
return v, ok, nil
})
for range 2 {
wout, werr := byWatch.doUpgrade(map[string]any{"compare": "true"}, "op")
if werr != nil {
t.Fatal(werr)
}
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
t.Errorf("compare by watch = %+v", got)
}
}
for _, state := range []string{"mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests"} {
if watches[state] != 0 || watches[state+" by key"] != 0 || indexer.asks[state] == 0 {
t.Errorf("the indexer's %s was watched %d times and read by key %d, asked of its first view %d times: want asked alone",
state, watches[state], watches[state+" by key"], indexer.asks[state])
}
}
for _, state := range []string{haveState, assign.FindingsState} {
if watches[state] != 1 {
t.Errorf("%s was watched %d times, want once", state, watches[state])
}
}
// ADR 0580 clause 3: the compare reads through mesh-sdk's watched reader and the seat keeps no mirror of its own.
for f := range reflect.TypeFor[wantsTools]().Fields() {
if f.Type.Kind() == reflect.Map || (f.Type.Kind() == reflect.Pointer && f.Type.Elem().Kind() == reflect.Map) {
t.Errorf("wantsTools keeps %s, a copy of its own beside the watched reader", f.Name)
}
}
if watches[policiesState] != 0 {
t.Errorf("the seat's own small policies state was watched: %v", watches)
}
// a person's record for web: written, recorded, said.
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
t.Fatal(err)
}
chosen, _ := assign.DecodePolicies(st[policiesState].values)
if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") ||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
}
// A person may choose the bus's policy as any module's (ADR 0601 §1: no part treats the module that provides the bus apart).
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err != nil {
t.Errorf("a person's policy for the bus was refused: %v", err)
}
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "default", "why": "back to the module"}, "op"); err != nil {
t.Fatal(err)
}
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
t.Error("a policy without why was set")
}
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 {
t.Errorf("default left %v (%v)", st[policiesState].values, err)
}
// bus: the node behind the indexer's newest.
b, err := w.doBus(nil)
if err != nil {
t.Fatal(err)
}
nodes := b.(map[string]any)["nodes"].([]busNode)
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
t.Errorf("bus = %+v, asked %v", b, asked)
}
// W5.5 (design 47 lines 1365-1366): every verb of the seat is served, revision, entry and difference reading only the
// kept revisions.
objects := memObjects{}
w.revisions = objects
tools := w.tools()
run := map[string]func(map[string]any) (any, error){}
for _, tool := range tools {
if tool.Run == nil {
t.Errorf("the seat's verb %s is not served", tool.Name)
}
run[tool.Name] = tool.Run
}
// the verb input is answered by the program in chunks (inputs.go), not as a tool
if len(tools) != 5 || tools[0].Name != "mesh-wants.upgrade" {
t.Errorf("the seat's verbs as served: %+v", tools)
}
if _, err := run["mesh-wants.revision"](map[string]any{}); !errors.Is(err, assigngraph.ErrNoRevision) {
t.Errorf("revision with none kept: %v", err)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{V: 1, Kind: "trunk", Placements: "sha256:p",
Catalogs: []meshwants.CatalogOf{{Branch: "main", Revision: 3}}, Entries: []meshwants.NodeEntryRef{{Node: "anchor", Module: "web", Digest: "sha256:e"}}},
Entries: map[string]meshwants.AssignmentEntry{"sha256:e": {Node: "anchor", Module: "web", Branch: "main", Fp: fpNew, Catalog: "sha256:c"}}}
if _, _, err := assigngraph.Keep(context.Background(), objects, c, "catalog-revision", time.Now()); err != nil {
t.Fatal(err)
}
r, err := run["mesh-wants.revision"](map[string]any{"revision": 1, "caller": "seat:mesh-grants"})
if err != nil || r.(meshwants.RevisionAnswer).Revision.Catalogs[0].Revision != 3 {
t.Errorf("revision = %+v, %v", r, err)
}
if e, err := run["mesh-wants.entry"](map[string]any{"node": "anchor", "module": "web"}); err != nil || e.(meshwants.EntryAnswer).Entry.Fp != fpNew {
t.Errorf("entry = %+v, %v", e, err)
}
if d, err := run["mesh-wants.difference"](map[string]any{"from": 1, "to": 1}); err != nil || len(d.(meshwants.Difference).Changes) != 0 {
t.Errorf("difference = %+v, %v", d, err)
}
}
// memObjects is an object bucket in memory.
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := m[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (m memObjects) Names(context.Context) ([]string, error) {
out := []string{}
for n := range m {
out = append(out, n)
}
return out, nil
}
// readManifest is the module's manifest, read as the controller reads it.
func readManifest() (catalogue.Manifest, error) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.Decode(raw)
}
// countingKV is a memKV that counts its puts.
type countingKV struct {
memKV
writes *int
}
func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
*c.writes++
return c.memKV.Put(ctx, key, value)
}
// revKV is a bucket with real revisions, refusing a write at a revision that is not the key's, and a write of refuse;
// log is every write in order.
type revKV struct {
jetstream.KeyValue
rev uint64
values map[string][]byte
revs map[string]uint64
log []string
refuse string
}
func (k *revKV) write(key string, value []byte) uint64 {
k.rev++
k.values[key], k.revs[key] = value, k.rev
return k.rev
}
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
if _, ok := k.values[key]; ok {
return 0, jetstream.ErrKeyExists
}
if key == k.refuse {
return 0, errors.New("refused")
}
k.log = append(k.log, "create "+key)
return k.write(key, value), nil
}
func (k *revKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
if k.revs[key] != revision || key == k.refuse {
return 0, errors.New("wrong last sequence")
}
k.log = append(k.log, "update "+key)
return k.write(key, value), nil
}
func (k *revKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
if key == k.refuse {
return errors.New("wrong last sequence")
}
k.log = append(k.log, "delete "+key)
delete(k.values, key)
delete(k.revs, key)
return nil
}
// historyStore is a memStore whose history is given.
type historyStore struct {
memStore
revisions map[string][]stdio.StateRevision
}
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
return s.revisions[key], nil
}
// putAssignments takes a node's assignments into the world as the keys of the state: a key per module, then the node key.
func putAssignments(w *world, a mma.Assignments) {
values, err := mmac.AssignmentsValues(a)
if err != nil {
panic(err)
}
for _, k := range slices.Sorted(maps.Keys(values)) {
w.putEntry(plc.AssignmentsBucket(), k, values[k], 1, time.Now())
}
}
// setAssignments holds a node's assignments in a state as its keys: a key per module, and the node key.
func setAssignments(kv *memKV, a mma.Assignments) {
values, err := mmac.AssignmentsValues(a)
if err != nil {
panic(err)
}
maps.Copy(kv.values, values)
}
// retiredKV is the retired state: its keys, its values, and deletes of them.
type retiredKV struct {
jetstream.KeyValue
values map[string][]byte
deletes int
}
func (k *retiredKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
if len(k.values) == 0 {
return nil, jetstream.ErrNoKeysFound
}
return slices.Sorted(maps.Keys(k.values)), nil
}
func (k *retiredKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := k.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return retiredEntry{key: key, value: v}, nil
}
func (k *retiredKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
k.deletes++
delete(k.values, key)
return nil
}
type retiredEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e retiredEntry) Value() []byte { return e.value }
func (e retiredEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
// emitted is a bus that keeps the module events it was asked to say.
type emitted struct {
caller
events []copiedSaid
}
func (e *emitted) Emit(eventType string, body any) error {
if eventType == CopiedEvent {
e.events = append(e.events, body.(copiedSaid))
}
return nil
}
func (k *revKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
if len(k.values) == 0 {
return nil, jetstream.ErrNoKeysFound
}
return slices.Sorted(maps.Keys(k.values)), nil
}
// copiesKV is the state copies: the marker of the end of a copy.
type copiesKV struct {
jetstream.KeyValue
values map[string][]byte
}
func (k *copiesKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
v, ok := k.values[key]
if !ok {
return nil, jetstream.ErrKeyNotFound
}
return retiredEntry{key: key, value: v}, nil
}
func (k *copiesKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
k.values[key] = value
return uint64(len(k.values)), nil
}
// fakeIndexer answers the indexer's first-view tools as its runtime does (views.FirstViewTool): each tool its state's values
// in key order, one key a page, each page continued after the page before's next; the tool named refused answers its
// refusal. asks counts the pages asked by tool.
type fakeIndexer struct {
views map[string]map[string][]byte
refused string
asks map[string]int
}
// answer is the answer to a call of key, and whether key is one of the indexer's first views at all.
func (f *fakeIndexer) answer(key string, body any) (json.RawMessage, bool, error) {
values, ok := f.views[key]
if !ok {
return nil, false, nil
}
if f.asks == nil {
f.asks = map[string]int{}
}
f.asks[key]++
if key == f.refused {
return nil, true, fmt.Errorf("%s is refused to \"novox.mesh-delivery-assigner\"", key)
}
after, _ := body.(map[string]any)["after"].(string)
a := views.Answer{V: views.Version, Owner: buckets.IndexerModule, State: strings.TrimPrefix(key, buckets.IndexerModule+"."), Revision: 7,
Values: map[string]json.RawMessage{}}
for _, k := range slices.Sorted(maps.Keys(values)) {
if k <= after {
continue
}
if len(a.Values) == 1 {
a.Next = slices.Collect(maps.Keys(a.Values))[0]
break
}
a.Values[k] = values[k]
}
raw, err := json.Marshal(a)
return raw, true, err
}
@@ -1,16 +0,0 @@
package main
import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// (plc.CheckAssignments), and seat-holders is not written.
// A call through the mesh MCP server is proved only as its machine's tool runner (issue 627), so the holders of the seat
// node-tool-runner stand for an operator's session until a session is proved (issue 1966). The list names seats, never a
// module, resolved from the assigner's own seat-holders (design 47 G A-210, A-295, A-298).
var seatListAgainCallers = []string{stdio.AnyPerson, stdio.CallerSeatPrefix + "node-tool-runner", stdio.CallerSeatPrefix + "mesh-healers"}
@@ -1,114 +0,0 @@
package main
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
)
// testDefinitions are seatlisttest's, parsed once.
var testDefinitions = sync.OnceValue(func() []seatdef.Definition { return seatlisttest.Definitions() })
// standInCatalog is the seat mesh-catalog as a test's reader asks it (ADR 0636 §4): every definition of seatlisttest, or,
// silent, no answer at all.
type standInCatalog struct{ silent bool }
func (c standInCatalog) Ask(key string, body any) (json.RawMessage, error) {
if c.silent || key != seatlist.Key() {
return nil, errors.New("no responders")
}
defs := testDefinitions()
seat, _ := body.(map[string]any)["seat"].(string)
if seat == "" {
l := seatlist.Listing{Revision: 1}
for _, d := range defs {
l.Seats = append(l.Seats, seatlist.Entry{Seat: d.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + d.Seat})
}
return json.Marshal(l)
}
for _, d := range defs {
if d.Seat == seat {
raw, err := json.Marshal(d)
if err != nil {
return nil, err
}
return json.Marshal(seatlist.Answer{Seat: seat, Found: true, Revision: 1,
Entry: &seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "d-" + seat, Definition: raw}})
}
}
return json.Marshal(seatlist.Answer{Seat: seat, Revision: 1})
}
func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
// gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list
// (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere.
func gateOver(t *testing.T, silent bool) *seatlist.Part {
t.Helper()
if silent {
// A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test
// that waits for it to stay unread keeps short.
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
}
g := seatlist.NewPart("assigner", seatListAgainCallers)
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
if r == nil {
t.Fatal("the reader did not start")
}
<-r.Ready()
if g.Heard() == silent {
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health())
}
return g
}
// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the
// assigner's code finds no start of a reader and no tools of its own.
func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} {
if strings.Contains(string(raw), own) {
t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own)
}
}
}
}
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
if _, ok := seatlist.Definition(seatlist.NewPart("assigner", seatListAgainCallers).Reader, "mesh-wants")(); ok {
t.Error("a definition is answered before the seat list's reader started")
}
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
t.Error("a definition is answered while the seat mesh-catalog has not answered the list")
}
read := gateOver(t, false)
if d, ok := seatlist.Definition(read.Reader, "mesh-wants")(); !ok || d.Seat != "mesh-wants" {
t.Errorf("the read list's mesh-wants is answered as %v, %v", d.Seat, ok)
}
if _, ok := seatlist.Definition(read.Reader, "no-such-seat")(); ok {
t.Error("a seat the list does not hold is answered a definition")
}
}
@@ -1,15 +1,9 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
plc "mesh-delivery-assigner/internal/assignment"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/puts"
"os"
"regexp"
"slices"
@@ -18,26 +12,14 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
// (seat_tools.go, ADR 0567 §10).
// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again.
const engineReadsTool = "assigner_engine_reads"
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
@@ -54,9 +36,7 @@ type store interface {
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
type putTools struct {
st store
// ask asks a module's tool (stdio.Ask, or a test's): the indexer's first view of modules for newest.
ask func(key string, body any) (json.RawMessage, error)
st store
now func() time.Time
mu sync.Mutex
// heard is every put the watch handed, by key; changed is closed and replaced at each.
@@ -64,8 +44,8 @@ type putTools struct {
changed chan struct{}
}
func newPutTools(st store, ask func(string, any) (json.RawMessage, error)) *putTools {
return &putTools{st: st, ask: ask, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
func newPutTools(st store) *putTools {
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
}
// hear takes a put's change in from the watch.
@@ -77,66 +57,72 @@ func (s *putTools) hear(key string, value []byte) {
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole.
var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181:
// a put names the whole fingerprint, the builder's builds being no bucket to list).
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
case v == "newest":
// Asked of the indexer's first view of modules, never read from its bus state (ADR 0596, ADR 0621; issue 1966).
mods, err := indexerView(s.ask, indexerModules)
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
if err != nil {
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
raw, found := mods[module]
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := meshcatalogclient.DecodeModule(raw)
m, err := delivery.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.Fp2 == "" {
if m.FP2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.Fp2, nil
return m.FP2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version)
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
puts.Put
delivery.Put
Follow string `json:"follow,omitempty"`
// Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the
// node's want until it ends or is released.
Older string `json:"older-than-newest,omitempty"`
// Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is
// written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*).
Since time.Time `json:"since"`
Bound string `json:"bound,omitempty"`
}
// answerOf is what assigner_put and assigner_puts answer of the put p of module on node.
func answerOf(node, module string, p puts.Put) putAnswer {
a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()}
if putWaits(p) {
a.Bound = putWaitBound.String()
}
return a
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -160,17 +146,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before.FP == fp && before.Hold == hold {
a := answerOf(node, module, before)
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := puts.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, puts.Asked, words)
data, err := puts.Encode(p)
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -178,21 +162,19 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
a := answerOf(node, module, s.await(key, p))
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p puts.Put) puts.Put {
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := puts.Decode(raw); err == nil && now.At.Equal(p.At) {
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != puts.Asked && now.Step != puts.Build && now.Step != puts.Compose {
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
return p // it waits on a person or the node, or ended
}
}
@@ -212,17 +194,8 @@ func followPut(node, module string) string {
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if arg(args, "how") != "" {
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
}
if step := arg(args, "step"); step != "" {
if step != meshwants.WaitStepDelivery {
return nil, fmt.Errorf("step is %s, the one wait released by its step, not %q", meshwants.WaitStepDelivery, step)
}
return s.doReleaseDelivery(args, by) // a dependent's delivery wait (delivered.go, design 47 G §376, A-246)
}
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -236,15 +209,15 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if puts.Ended(p.Step) {
if delivery.PutEnded(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), puts.Released, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := puts.Encode(p)
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -252,7 +225,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return answerOf(node, module, p), nil
return putAnswer{Node: node, Module: module, Put: p}, nil
}
// doPuts is the tool assigner_puts.
@@ -263,10 +236,8 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
}
sort.Strings(keys)
out := []putAnswer{}
newest := map[string]string{} // the indexer's newest fingerprint by module, asked once and only for a put that stands
asked := false
for _, key := range keys {
node, module, err := puts.ParseKey(key)
node, module, err := delivery.ParsePutKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
@@ -274,27 +245,11 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
continue
}
a := answerOf(node, module, p)
if !puts.Ended(p.Step) {
if !asked {
asked = true
if mods, err := indexerView(s.ask, indexerModules); err == nil {
for name, raw := range mods {
if m, err := meshcatalogclient.DecodeModule(raw); err == nil {
newest[name] = m.Fp2
}
}
}
}
if n := newest[module]; n != "" && n != p.FP {
a.Older = n
}
}
out = append(out, a)
out = append(out, putAnswer{Node: node, Module: module, Put: p})
}
return map[string]any{"puts": out}, nil
}
@@ -351,14 +306,9 @@ func (s *putTools) tools() []stdio.Tool {
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
"failed (the failing check is the delivery's), rejudge (judge it again now), or now (the entry waiting for an acting part's new version moves with the running one); written in the hand-act log first",
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
[]string{"module", "why"}),
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
[]string{"node", "module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
s.waitsTool(),
s.deliveredAgainTool(),
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
@@ -386,8 +336,7 @@ func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(st
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
func serveTools() error {
plc.SetModule(os.Getenv(stdio.ServedAs)) // the module the runtime serves these tools as (A-296)
s := newPutTools(runtimeStore{}, stdio.Ask)
s := newPutTools(runtimeStore{})
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
@@ -402,38 +351,12 @@ func serveTools() error {
time.Sleep(wait)
}
}()
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
gate := seatlist.NewPart("assigner", seatListAgainCallers)
go func() {
<-stdio.Served()
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
}()
seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask)
seatVerbs.seatList = gate
tools := append(s.tools(), seatVerbs.tools()...)
tools = append(tools, gate.Tools()...)
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
return stdio.SeatPublish(wantsEventSubject(e), b, id)
})
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
tools = append(tools, wants.tools()...)
tools = append(tools, inputsRead(&inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}))
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
tools = append(tools, writtenTools(runtimeStore{})...)
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
return stdio.Serve("", tools)
return stdio.Serve("", s.tools())
}
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := catalogkey.KeyPart(fp)
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
@@ -1,110 +0,0 @@
package main
import (
"slices"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
// manifests, as a provider is counted (ADR 0603 §5).
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
// no node of module.
func keptInvokes(g graph.Graph, module string) []string {
n, ok := g[module]
if !ok {
return nil
}
var out []string
for _, v := range n.Invokes {
if slices.Contains(n.LeftOut, v) {
continue
}
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
out = append(out, v)
}
}
return out
}
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
// a verb.
func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
if len(g) == 0 {
return nil
}
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
}
return served
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
// that build serves what it offers (A-352); a consumer needing the newer build's forms waits by its kind-4 edge instead. A
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
// holds a consumer for good.
func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
offered := map[string]bool{}
for node, haves := range n.Haves {
if !n.Online[node] {
continue
}
for module, h := range haves {
if h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseWaitingYes || h.Health != nodeengineclient.HealthOK {
continue
}
m, ok := p.manifestOf(module, h.Fp)
if !ok {
m = trunk[module]
}
for _, o := range m.Offers() {
offered[o] = true
}
}
}
return func(provision string) bool { return offered[provision] }
}
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
func (p *passer) servesOf(module, fp string) ([]string, bool) {
m, ok := p.manifestOf(module, fp)
if !ok {
return nil, false
}
return graph.ServesOf(m), true
}
// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests
// hold), and whether it could be read.
func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
e, ok := p.w.ends.endOf(module, fp)
if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" {
return catalogue.Manifest{}, false
}
got := p.w.manifest(e.Manifest)
if got.err != "" {
return catalogue.Manifest{}, false
}
return got.manifest, true
}
@@ -1,196 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// marksBucket is the assigner's state of what its views family said each state up to.
func marksBucket() string { return assignment.Module() + "_" + views.MarksState }
// viewedTool is one tool over one state of the assigner's.
type viewedTool struct{ tool, state, words string }
// firstViewTools are the assigner's first-view tools (ADR 0621 §1): each answers only its callers list
// (views.CallersEnv), which the manifest writes.
var firstViewTools = []viewedTool{
{"assignments", mma.StateAssignments, "where each module is placed: the seat mesh-module-assigner's assignments, by module."},
{"seat-holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"wants", "wants", "what each node should run: want.<node>.<module>, its fingerprint and the rule that gave it."},
// The seat mesh-wants' asks (design 47 G §717): the asks the assigner put to a person about what a node runs.
{"asks", "asks", "the asks the assigner put to a person about what a node runs, and their answers."},
}
// readTools are the assigner's ordinary read tools, one per state its readers read (design 25 § 12.2).
var readTools = []viewedTool{
{"assigner_read_assignments", mma.StateAssignments, "where each module is placed, by module."},
{"assigner_read_seat_holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
{"assigner_read_findings", assign.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
{"assigner_read_copies", copiesState, "the copies the assigner made of a retired state: when the copy into assignments ended and how many keys it copied."},
}
// viewedStates are the states the views family says, each by its bucket: the six its readers read.
func viewedStates() map[string]string {
return map[string]string{
mma.StateAssignments: plc.AssignmentsBucket(),
mma.StateSeatHolders: plc.SeatHoldersBucket(),
"wants": buckets.Wants(),
assign.FindingsState: assign.FindingsBucket(),
"asks": buckets.Asks(),
"differences": buckets.Differences(),
copiesState: copiesBucket(),
}
}
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
// runtime of its own is connected (ADR 0621 §3).
func viewTools(getenv func(string) string, state func(string) views.Source, now func() time.Time) []stdio.Tool {
marks := state(views.MarksState)
var out []stdio.Tool
for _, t := range firstViewTools {
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, views.ParseCallers(getenv(views.CallersEnv(t.tool))), state(t.state), marks, now))
}
for _, t := range readTools {
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
}
// Its own first views, of the indexer's manifests, modules and graph (DLV-F3r, ADR 0631 §2): read, and asked again by hand.
return append(out, views.StatusTools(assignment.Module(), "assigner_first_views", "assigner_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
}
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, writer views.WireWriter, errs chan<- error, logf func(string, ...any)) {
process.AwaitBucket(ctx, js, nc, marksBucket(), "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
em := viewsEmitter(js, account, mk, writer, logf)
for _, state := range []string{"asks", "differences", assign.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
state := state
start := func(kv jetstream.KeyValue) {
if err := em.Start(ctx, state, kv, errs); err != nil {
logf("%v: its readers ask its first view at each change until the assigner starts again", err)
}
}
if kv := kvs[viewedStates()[state]]; kv != nil {
start(kv)
continue
}
process.AwaitBucket(ctx, js, nc, viewedStates()[state], "its changes are not said on the views family", logf).Ready(start)
}
})
}
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
// view-unsaid, in its findings, written through its finding writer (review round 2); with none it is said in the log only.
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: assignment.Module(), Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logf}
}
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
func firstViewsBucket() string { return assignment.Module() + "_" + views.StatusState }
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195, A-26): the
// module's views reader adopts each branch's follower, so it asks the seat mesh-catalog's verb `revision` (the trunk's, then
// each branch a registration names), follows its event `catalog-revision`, asks once more on a gap, and past the bound raises
// the finding naming the seat and the verb, cleared by the answer, in the assigner's findings. Its status is read by
// assigner_first_views and asked again by assigner_first_view_again.
type catalogViews struct {
js jetstream.JetStream
account process.Account
call func(string, any) (json.RawMessage, error)
writer views.WireWriter
status *process.AwaitedBucket
logf func(string, ...any)
hub *process.EventHub
h *halves
// candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made;
// before that a candidate's revision is said not taken.
candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error]
// answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers,
// then its views family; nil when it is not followed.
answers *views.Picture
}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
// seconds). A test shortens them.
var firstViewBound, firstViewWait time.Duration
// start adopts the trunk's catalog half through the reader and starts the reader; it answers how many settles to wait for.
func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
reader := &views.ProcessReader{Module: assignment.Module(), Call: iv.call, Logf: iv.logf, Hub: iv.hub, Bound: firstViewBound, Wait: firstViewWait}
if iv.writer != nil {
reader.Findings = func() views.WireWriter { return iv.writer }
}
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
if err := iv.h.follow(); err != nil {
iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err)
}
// Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185).
if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
if f := iv.candidates.Load(); f != nil {
return (*f)(ctx, ev)
}
return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision)
}); err != nil {
iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err)
}
// The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket.
if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil {
iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err)
} else {
iv.answers = pic
}
reader.Start(ctx, iv.js, iv.account, iv.status)
return 1
}
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
// findings state it is said in the log alone.
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
w := heard.Writer{Observer: assignment.Module(), Logf: logf}
if findings.Put != nil {
w.Put = func(_ string, value []byte) error {
var f mcseat.Finding
if err := json.Unmarshal(value, &f); err != nil {
return err
}
return findings.Put(f)
}
}
if findings.Delete != nil {
w.Delete = func(key string) error {
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
return nil // none stands
}
return findings.Delete(key)
}
}
return w
}
@@ -1,332 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strconv"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-module-assigner's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
const (
policiesState = buckets.PoliciesState
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
busStepCause = "bus-upgrade"
)
// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit.
type wantsTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
emit func(event, id string, body any) error
now func() time.Time
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
// a large state (issue 1953 L5f). Nil reads key by key.
watched *stdio.WatchedStates
// revisions is the kept data assignment-revisions the verbs revision, entry and difference read (design 47 lines
// 1365-1366); nil serves them not.
revisions assigngraph.Objects
}
// wantsEventSubject is the subject of one of the seat's events.
func wantsEventSubject(event string) string {
return "mesh.seat." + meshwants.Seat + ".event." + event
}
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
}
// values reads every key of a state: from the watched copy when there is one, else key by key.
func (s *wantsTools) values(state string) (map[string][]byte, error) {
if s.watched != nil {
return s.watched.Values(state)
}
return s.direct(state)
}
// direct reads a state key by key, for a small state of the seat's own.
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
keys, err := s.st.keys(state)
if err != nil {
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
}
out := map[string][]byte{}
for _, k := range keys {
raw, _, found, err := s.st.get(state, k)
if err != nil {
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
}
if found {
out[k] = raw
}
}
return out, nil
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint: both asked of the indexer's first
// views of modules and manifests (indexerView), never read from its bus state.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]meshcatalog.ModulesByModuleValue, error) {
mods, err := indexerView(s.ask, indexerModules)
if err != nil {
return nil, nil, err
}
kept, err := indexerView(s.ask, indexerManifests)
if err != nil {
return nil, nil, err
}
out, held := map[string]catalogue.Manifest{}, map[string]meshcatalog.ModulesByModuleValue{}
for name, raw := range mods {
m, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
continue
}
held[name] = m
if m.Manifest == "" {
continue
}
mraw, found := kept[m.Manifest]
if !found {
continue
}
stored, err := meshcatalogclient.DecodeManifest(mraw)
if err != nil {
continue
}
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
if man, err := catalogue.Decode([]byte(stored.Manifest)); err == nil {
out[name] = man
}
}
return out, held, nil
}
// policyRow is one module's policy as upgrade answers it.
type policyRow struct {
Module string `json:"module"`
Policy string `json:"policy"`
From string `json:"from"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
}
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
// back with default; with compare each node's composition and the assigner's findings.
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
manifests, _, err := s.manifests()
if err != nil {
return nil, err
}
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
if err != nil {
return nil, err
}
chosen, unreadable := assign.DecodePolicies(raw)
if arg(args, "compare") == "true" {
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(buckets.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(assign.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
keys := []string{}
for k := range found {
keys = append(keys, k)
}
sort.Strings(keys)
out["findings"] = keys
}
return out, nil
}
module, policy := arg(args, "module"), arg(args, "policy")
if policy != "" {
return s.setPolicy(module, policy, arg(args, "why"), by, manifests)
}
var rows []policyRow
for name, m := range manifests {
if module != "" && name != module {
continue
}
p, from, why, who := assign.PolicyOf(m, chosen)
rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who})
}
if module != "" && len(rows) == 0 {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
return map[string]any{"modules": rows, "unreadable": unreadable}, nil
}
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
if module == "" {
return nil, errors.New("a policy is set for one module: name it")
}
if _, ok := manifests[module]; !ok {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
if why == "" {
return nil, errors.New("a person's policy says why")
}
if by == "" || by == stdio.Unproven {
return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller")
}
what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy)
if policy == "default" {
if err := s.st.del(policiesState, module); err != nil {
return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err)
}
} else {
p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()}
if err := p.Check(); err != nil {
return nil, err
}
raw, _ := json.Marshal(p)
if err := s.st.put(policiesState, module, raw); err != nil {
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
}
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, rerr := rec.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
}
body := map[string]any{"module": module, "policy": policy, "why": why, "by": by}
said := ""
if s.emit != nil {
id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10)
if err := s.emit(PolicyChangedEvent, id, body); err != nil {
said = "not said: " + err.Error()
}
}
return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil
}
// busNode is one node's bus as bus answers it.
type busNode struct {
Node string `json:"node"`
FP string `json:"fp"`
Behind bool `json:"behind"`
}
// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind,
// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only.
func (s *wantsTools) doBus(map[string]any) (any, error) {
manifests, held, err := s.manifests()
if err != nil {
return nil, err
}
bus := ""
for name, m := range manifests {
if catalogue.ProvidesBus(m) {
bus = name
}
}
if bus == "" {
return nil, errors.New("the indexer holds no module that provides the bus")
}
newest := held[bus].Fp2
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
if steps, err := s.ask("seat:"+meshhandacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
out["steps-unread"] = err.Error()
} else {
out["steps"] = steps
}
kept, err := s.values(putsState)
if err != nil {
out["puts-unread"] = err.Error()
} else {
mine := map[string]json.RawMessage{}
for k, v := range kept {
if _, module, err := puts.ParseKey(k); err == nil && module == bus {
mine[k] = v
}
}
out["puts"] = mine
}
return out, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>; the tool runner describes them with the
// seat's definition from the seat list (ADR 0635 §1), so the tools carry none.
func (s *wantsTools) tools() []stdio.Tool {
run := map[string]func(map[string]any) (any, error){
meshwants.VerbUpgrade: func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
meshwants.VerbBus: s.doBus,
}
if s.revisions != nil {
for v, f := range graphVerbs(s.revisions) {
run[v] = f
}
}
var out []stdio.Tool
for _, v := range meshwants.Verbs() {
if run[v] == nil {
continue // a verb of the seat this module does not serve yet: revision, entry, difference (AS11)
}
out = append(out, stdio.Tool{Name: meshwants.Seat + "." + v, Run: run[v]})
}
return out
}
@@ -1,32 +1,41 @@
package main
import (
"context"
"encoding/json"
"errors"
"maps"
"mesh-delivery-assigner/internal/buckets"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/keys"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
const noModuleWords = delivery.CandidateNoneNoModule
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
// internal/view/view.go): an assignment stands while it has not ended and has no state.
type assignment struct {
Node string `json:"node"`
Module string `json:"module"`
Build string `json:"build"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
// digest. The watches write it; a pass reads it.
type world struct {
@@ -37,41 +46,6 @@ type world struct {
healthy map[string][]string
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
changes map[string]uint64
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
// ends are the done builds' ends the builder says (build.ended), by module and fingerprint (buildlog.go).
ends builtEnds
// released are the catalog's version-released heard, by module and source revision (buildended.go).
released releasedVersions
// delivered are the assigner's own version-delivered heard and said (delivered.go).
delivered deliveredVersions
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
edges map[string]map[string][]meshcatalog.Edge
// taken is the newest source revision of each watched branch the indexer's take-in.taken named, as heard (orderwaits.go, §298, A-260).
taken takenSources
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
assignmentKeys *mmac.Keys
// first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the
// branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532),
// and never fixed in code (§7).
first string
// presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389).
presence map[string]presenceSaid
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
running map[string]nodeengine.Running
runningHeard bool
// modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14,
// §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog.
modules map[string]map[string]meshcatalog.ModulesByModuleValue
graphs map[string]map[string]graph.Node
}
// entryMeta is a value's revision in its bucket and when it was written.
type entryMeta struct {
revision uint64
at time.Time
}
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
@@ -82,32 +56,7 @@ type parsed struct {
}
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(),
running: map[string]nodeengine.Running{},
modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}}
}
// putEntry is put, keeping the value's revision and the time it was written.
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
w.put(bucket, key, value)
w.mu.Lock()
defer w.mu.Unlock()
if value == nil {
delete(w.meta, bucket+"|"+key)
return
}
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision
}
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
func (w *world) metaOf(bucket, key string) entryMeta {
w.mu.Lock()
defer w.mu.Unlock()
return w.meta[bucket+"|"+key]
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
}
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
@@ -117,31 +66,23 @@ func (w *world) put(bucket, key string, value []byte) {
if w.values[bucket] == nil {
w.values[bucket] = map[string][]byte{}
}
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
if value == nil {
delete(w.values[bucket], key)
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, nil, 0, true)
}
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == plc.AssignmentsBucket() {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.assignmentKeys.Put(key, value, 0, false)
}
if bucket == buckets.Have {
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
w.healthy[key] = append(list, h.Fp)
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
w.healthy[key] = append(list, h.FP)
}
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := nodeengineclient.HaveKey(node, module)
key, err := delivery.HaveKey(node, module)
if err != nil {
return ""
}
@@ -167,237 +108,177 @@ func (w *world) read(bucket string) map[string][]byte {
return out
}
// getter reads one key's value; errNotThere when the key does not exist.
type getter func(ctx context.Context, key string) ([]byte, error)
var errNotThere = errors.New("no such key")
// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable.
func (w *world) manifest(digest string) parsed {
// kvGetter reads keys of a bucket.
func kvGetter(kv jetstream.KeyValue) getter {
return func(ctx context.Context, key string) ([]byte, error) {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, errNotThere
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
}
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
w.mu.Lock()
defer w.mu.Unlock()
if p, ok := w.manifests[digest]; ok {
p, ok := w.manifests[digest]
w.mu.Unlock()
if ok {
return p
}
return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"}
}
// assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the
// assignments is the one source of where modules run. The controller's assignments are read no more (issue 774).
func (w *world) assigned() map[string][]string {
placed, _ := w.placed()
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
key, err := delivery.ManifestsKey(digest)
if err != nil {
return parsed{err: err.Error()}
}
return out
}
// placed is each node's modules in the seat's assignments (ADR 0567 §1), and whether any assignments key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
for node, p := range w.assignments() {
out[node] = mmac.Assigned(p)
value, err := get(ctx, key)
if errors.Is(err, errNotThere) {
return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"}
}
return out, len(w.read(plc.AssignmentsBucket())) > 0
}
// assignments is every node's assignments, assembled from its keys in either form (mmac.Keys).
func (w *world) assignments() map[string]mma.Assignments {
out, _ := w.assignmentKeys.AllRevisions()
return out
}
// placedNodes is every node the seat's assignments names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
out := make(map[string]bool, len(placed))
for node := range placed {
out[node] = true
if err != nil {
return parsed{err: err.Error()}
}
m, err := delivery.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
p = parsed{module: m.Module, err: err.Error()}
} else {
p = parsed{module: m.Module, manifest: man}
}
return out
}
// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184).
func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue {
w.mu.Lock()
defer w.mu.Unlock()
return maps.Clone(w.modules[branch])
w.manifests[digest] = p
w.mu.Unlock()
return p
}
// graphOf is the module graph of the catalog half of branch.
func (w *world) graphOf(branch string) graph.Graph {
w.mu.Lock()
defer w.mu.Unlock()
return graph.Graph(maps.Clone(w.graphs[branch]))
}
// branches are the watched branches the catalog half is held of, the trunk's first.
func (w *world) branches() []string {
w.mu.Lock()
defer w.mu.Unlock()
out := slices.Sorted(maps.Keys(w.modules))
if i := slices.Index(out, w.first); i > 0 {
out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first)
}
return out
}
// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's
// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it.
func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue {
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, b := range w.branches() {
for name, v := range w.trunkOf(b) {
if _, ok := out[name]; !ok {
out[name] = v
}
}
}
return out
}
// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the
// value names.
func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) {
w.mu.Lock()
defer w.mu.Unlock()
if w.modules[branch] == nil {
w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{}
}
w.modules[branch][module], w.graphs[branch][module] = v, n
if m != nil && v.Manifest != "" {
w.manifests[v.Manifest] = *m
}
}
// dropModule takes a module out of branch's catalog half.
func (w *world) dropModule(branch, module string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.modules[branch], module)
delete(w.graphs[branch], module)
delete(w.edges[branch], module)
}
// putEdges keeps the edges into module's artifacts that branch's catalog entry names.
func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) {
w.mu.Lock()
defer w.mu.Unlock()
if w.edges == nil {
w.edges = map[string]map[string][]meshcatalog.Edge{}
}
if w.edges[branch] == nil {
w.edges[branch] = map[string][]meshcatalog.Edge{}
}
w.edges[branch][module] = edges
}
// edgesOf is the edges into module's artifacts in branch's catalog half.
func (w *world) edgesOf(branch, module string) []meshcatalog.Edge {
w.mu.Lock()
defer w.mu.Unlock()
return w.edges[branch][module]
}
// holdsBranch says whether the catalog half of branch has been taken in (even empty).
func (w *world) holdsBranch(branch string) bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.modules[branch] != nil
}
// records is each node's record in its own have key (`<node>.node`): the nodes an assignments proposal is answered by,
// with those assignments names (ADR 0593 §8).
func (w *world) records() map[string]nodeengine.NodeRecord {
out := map[string]nodeengine.NodeRecord{}
for key, value := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
func (w *world) assigned() (map[string][]string, map[string]assignment) {
out, all := map[string][]string{}, map[string]assignment{}
for _, value := range w.read(provider.AssignmentsBucket) {
var a assignment
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
continue
}
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
out[a.Node] = append(out[a.Node], a.Module)
all[a.Node+"."+a.Module] = a
}
return out
for _, mods := range out {
sort.Strings(mods)
}
return out, all
}
// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389).
type presenceSaid struct {
away bool
at time.Time
}
// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two
// streams, read whole in either order, leave each machine as its last event says.
func (w *world) heardPresence(machine string, away bool, at time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
if w.presence == nil {
w.presence = map[string]presenceSaid{}
}
if was, ok := w.presence[machine]; ok && at.Before(was.at) {
return
}
w.presence[machine] = presenceSaid{away: away, at: at}
}
// awayNow is every machine whose last presence event heard is machine-away.
func (w *world) awayNow() map[string]bool {
w.mu.Lock()
defer w.mu.Unlock()
out := map[string]bool{}
for m, p := range w.presence {
if p.away {
out[m] = true
}
}
return out
}
// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to
// the seat's verb is heard (novox/hq ADR 0668 §3).
func (w *world) seatHeard(seat string) bool {
online := w.online()
for _, value := range w.read(plc.SeatHoldersBucket()) {
var sh mma.SeatHolders
if json.Unmarshal(value, &sh) != nil || sh.Seat != seat {
continue
}
for _, h := range sh.Holders {
if online[h.Node] {
return true
}
}
}
return false
}
// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its
// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2).
func (w *world) online() map[string]bool {
away := w.awayNow()
out := map[string]bool{}
for _, node := range w.knownNodes() {
out[node] = !away[node]
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]mwc.Want {
out := map[string]map[string]mwc.Want{}
for key, value := range w.read(buckets.Wants()) {
node, module, err := mwc.ParseWantKey(key)
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
for key, value := range w.read(delivery.ModulesBucket) {
name, err := delivery.ParseModulesKey(key)
if err != nil {
continue
}
want, err := mwc.DecodeWant(value)
if m, err := delivery.DecodeModule(value); err == nil {
out[name] = m
}
}
return out
}
// mesh is the mesh as the trunk holds it, for the composition check.
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
m.Assigned, _ = w.assigned()
for name, mod := range w.trunk() {
if mod.Manifest == "" {
continue
}
m.Digests[name] = mod.Manifest
p := w.manifest(ctx, manifests, mod.Manifest)
if p.err != "" {
m.Unreadable[name] = p.err
continue
}
m.Manifests[name] = p.manifest
}
return m
}
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
definedIn := map[string]map[string]bool{}
for _, p := range c.Pulls {
if p.Repository == assign.SDKRepository {
ch.SDKHead = p.Head.Head
}
ch.Members[p.Repository] = p.Head.Head
for _, name := range p.Modules {
fp, ok := p.Fps[name]
if !ok || fp.Manifest == "" {
if ok && fp.None2 == noModuleWords {
ch.Removed = append(ch.Removed, name)
continue
}
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
}
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
ch.Rebuilt++
}
got := w.manifest(ctx, manifests, fp.Manifest)
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
return ch, got.err
}
ch.Digests[name] = fp.Manifest
if got.err != "" {
ch.Unreadable[name] = got.err
continue
}
ch.Manifests[got.module] = got.manifest
if definedIn[got.module] == nil {
definedIn[got.module] = map[string]bool{}
}
definedIn[got.module][p.Repository] = true
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
definedIn[got.module][t.Repository] = true
}
}
}
for name, repos := range definedIn {
if len(repos) > 1 {
for r := range repos {
ch.Twice[name] = append(ch.Twice[name], r)
}
sort.Strings(ch.Twice[name])
}
}
return ch, ""
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
for key, value := range w.read(delivery.AssignerWantsBucket) {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
continue
}
want, err := delivery.DecodeWant(value)
if err != nil {
continue
}
if out[module] == nil {
out[module] = map[string]mwc.Want{}
out[module] = map[string]delivery.Want{}
}
out[module][node] = want
}
@@ -406,24 +287,24 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
n.Online = w.online()
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if isNode {
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
n.Reads[node] = h.Record.ManifestReads
}
if p, err := delivery.DecodePresence(value); err == nil {
n.Online[node] = p.Online
}
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
continue
}
if h, err := nodeengineclient.DecodeHave(value); err == nil {
if h, err := delivery.DecodeHave(value); err == nil {
if n.Haves[node] == nil {
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
n.Haves[node] = map[string]delivery.Have{}
}
n.Haves[node][module] = h
}
@@ -452,10 +333,6 @@ func (w *world) value(bucket, key string) []byte {
return w.values[bucket][key]
}
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
// manifest but the controller's reads the controller's grants.
func (w *world) grantsBucket() string { return buckets.Grants }
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
w.mu.Lock()
@@ -464,84 +341,66 @@ func (w *world) has(bucket, key string) bool {
return ok
}
// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181).
// built says whether the builder's build of a module at a fingerprint is done.
func (w *world) built(module, fp string) bool {
e, ok := w.ends.endOf(module, fp)
return ok && e.Outcome == builds.BuildDone
}
// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G
// §575, A-162): the words of a hold under rule 1.
func (w *world) buildStanding(module, fp string) (failed bool, words string) {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed {
return true, e.Error
}
return false, ""
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR
// 0618 §1, A-181): nil when it is not done or names none.
func (w *world) buildNeeds(module, fp string) []string {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone {
return e.Needs
}
return nil
}
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
func (w *world) recordReads(node string) ([]string, bool) {
r, ok := w.records()[node]
return r.ManifestReads, ok
}
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
func (w *world) knownNodes() []string {
set := map[string]bool{}
for node := range w.placedNodes() {
set[node] = true
}
for node := range w.records() {
set[node] = true
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return false
}
w.mu.Lock()
for node := range w.presence {
set[node] = true
}
value, ok := w.values[delivery.BuilderBuildsBucket][key]
w.mu.Unlock()
return slices.Sorted(maps.Keys(set))
if !ok {
return false
}
b, err := delivery.DecodeBuilderBuild(value)
return err == nil && b.State == delivery.BuildDone
}
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
if !w.placedNodes()[node] {
return assign.ComposedDigest{}, false
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
w.mu.Lock()
value, ok := w.values[delivery.ControllerComposedBucket][key]
w.mu.Unlock()
if !ok {
return delivery.ComposedDigest{}, false
}
d, err := delivery.DecodeComposedDigest(value)
return d, err == nil
}
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
continue
}
if d, err := delivery.DecodeComposedDigest(value); err == nil {
out[node] = d
}
}
return out
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (mdc.Answer, bool) {
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[buckets.Answers][key]
value, ok := w.values[delivery.AnswersBucket][key]
w.mu.Unlock()
if !ok {
return mdc.Answer{}, false
return delivery.Answer{}, false
}
a, err := mdc.DecodeAnswer(value)
a, err := delivery.DecodeAnswer(value)
return a, err == nil
}
// bucketGen is how many times any key of the bucket was written or deleted in this process: a read of what it shows
// elsewhere is asked again only after it moves.
func (w *world) bucketGen(bucket string) uint64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.changes[bucket]
}
// changed is how many values the key has taken in this process (see world.changes).
func (w *world) changed(bucket, key string) uint64 {
w.mu.Lock()
@@ -549,64 +408,30 @@ func (w *world) changed(bucket, key string) uint64 {
return w.changes[bucket+"|"+key]
}
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
key, err := nodeengineclient.HaveNodeKey(node)
if err != nil {
return assign.ComposedDigest{}, false
}
raw, ok := w.read(buckets.Have)[key]
if !ok {
return assign.ComposedDigest{}, false
}
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return assign.ComposedDigest{}, false
}
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
Words: "the node composes its own"}, true
}
// unplacedNodes are the nodes whose engine has a have and that no assignments names: they get no
// want, since nothing composes them.
func (w *world) unplacedNodes() []string {
placed := w.placedNodes()
var out []string
for key := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
_, all := w.assigned()
seen := map[string]bool{}
for _, a := range all {
if a.Module != module || a.Build == "" {
continue
}
value, err := get(ctx, delivery.BuildKey(module, a.Build))
if err != nil {
continue
}
var rec delivery.BuildRecord
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
continue
}
seen[rec.Commit] = true
}
out := make([]string, 0, len(seen))
for c := range seen {
out = append(out, c)
}
sort.Strings(out)
return out
}
// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted.
func (w *world) trunkName() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.first
}
// setTrunk says which watched branch is the first.
func (w *world) setTrunk(branch string) {
w.mu.Lock()
defer w.mu.Unlock()
w.first = branch
}
// placedBranches is, by module and node, the watched branch each placement names ("" when it names none).
func (w *world) placedBranches() map[string]map[string]string {
out := map[string]map[string]string{}
for node, a := range w.assignments() {
for _, m := range a.Modules {
if out[m.Module] == nil {
out[m.Module] = map[string]string{}
}
out[m.Module][node] = m.Branch
}
}
return out
}
@@ -2,17 +2,14 @@ package main
import (
"context"
"mesh-delivery-assigner/internal/buckets"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// fakeManifests is the indexer's manifests bucket in memory.
@@ -20,8 +17,8 @@ type fakeManifests map[string][]byte
func (f fakeManifests) add(t *testing.T, module, body string) string {
t.Helper()
digest := meshcatalogclient.ManifestDigest([]byte(body))
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
digest := delivery.ManifestDigest([]byte(body))
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
if err != nil {
t.Fatal(err)
}
@@ -36,79 +33,95 @@ func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
return nil, errNotThere
}
func fp(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
for m := range modules {
p.Modules = append(p.Modules, m)
}
if got := w.changed(buckets.Have, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
slices.Sort(p.Modules)
return p
}
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
// another repository as defined twice.
func TestTheChangeOfACandidate(t *testing.T) {
f := fakeManifests{}
web := f.add(t, "web", `{"module":"web","version":"2"}`)
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
pull(assign.SDKRepository, 3, nil),
}}
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
if waits != "" {
t.Fatalf("the change waits: %s", waits)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name string
inPart bool
granted bool
}{
{"the part's grant", true, true},
{"no grant", false, false},
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
t.Errorf("change = %+v", ch)
}
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
}
}
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
// saying for what; an unreadable manifest is the change's, not a wait.
func TestAChangeWaitsForItsManifests(t *testing.T) {
f := fakeManifests{}
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
for _, c := range []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
} {
p := newPasser(kvSet{}, time.Now())
if row.inPart {
p.w.put(buckets.Grants, gkey, grant)
}
if got := p.w.grantsBucket(); got != buckets.Grants {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
if waits == "" {
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
}
}
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
if waits != "" || ch.Unreadable["web"] == "" {
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
}
}
// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it.
func putModuleRaw(w *world, branch, module string, raw []byte) {
v, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
panic(err)
}
w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil)
}
// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the
// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main".
func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) {
w := newWorld()
w.setTrunk("develop")
for _, b := range []string{"beta", "develop", "alpha"} {
w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil)
}
if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" {
t.Errorf("branches = %v, want the first watched first, then by name", got)
}
files, err := filepath.Glob("*.go")
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
// xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(src), `"main"`) {
t.Errorf("%s names the branch main in code (design 47 G §507)", f)
}
if !strings.Contains(string(raw), `"predefined"`) {
t.Fatal("the fixture names no predefined key")
}
f := fakeManifests{}
xterm := f.add(t, "xterm", string(raw))
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
}
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
}
+5 -95
View File
@@ -3,110 +3,20 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.1 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.2 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.16.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk => ../mesh-delivery-sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient => ../mesh-delivery-contracts/go/clients/meshcatalogclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient => ../mesh-delivery-contracts/go/clients/meshdeliveryclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog => ../mesh-delivery-contracts/go/seats/meshcatalog
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery => ../mesh-delivery-contracts/go/seats/meshdelivery
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks => ./asks
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents => ../mesh-delivery-teller/tellerevents
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate => ../mesh-delivery-view/viewstate
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient => ../mesh-delivery-contracts/go/clients/meshdeliverycheckerclient
require git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient => ../mesh-delivery-contracts/go/clients/meshpipelinerunnerclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner => ../mesh-delivery-contracts/go/seats/meshpipelinerunner
+18 -8
View File
@@ -1,13 +1,23 @@
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
@@ -21,5 +31,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
@@ -1,140 +0,0 @@
// Package acting is the assigner's wait for an acting part's new version (design 47 G §266, §268, §276): an entry of the assignment
// graph whose edge of kind acting names a step the assigner holds (composing, placing) waits until the acting part's version runs
// on the node of the edge, with the bound of the rule that gave the edge, or the step's default. The package holds the rules and
// the kept record; the assigner's process reads what runs, says the events and keeps the records in its state `waits`.
package acting
import (
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// HowNow is the `how` of assigner_release for a wait for an acting part: the entry is wanted now, with the version running.
const HowNow = "now"
// The ways a wait ends, as `wait-ended` says them.
const (
HowRan = "ran"
HowReleased = "released"
HowReplaced = "replaced"
)
// Prefix opens the key of an acting wait in the state, apart from the outside waits' `<node>.<module>.<fingerprint hex>`.
const Prefix = "acting."
// OrderPrefix opens the key of an order wait (design 47 G §298, A-259): a want held for the order of a delivery that crosses a form change,
// kept beside the acting waits in the state `waits`, as `order.<node>.<module>.<part>`.
const OrderPrefix = "order."
// The three order edges of §298, as a kept wait's Order names them.
const (
// OrderBackup: the new versions of the node-backup holder and of the holder of the bus module's seat run before a module whose kept
// data's form changes is moved.
OrderBackup = "backup-and-broker-first"
// OrderEngineLast: at a revert a node's node-engine goes back last on that node.
OrderEngineLast = "node-engine-last"
// OrderTakenIn: the reverted assigner and gate are wanted only once the reverted indexer took in the trunk's head.
OrderTakenIn = "taken-in-first"
)
// Held says whether the assigner holds the entries of step: composing and placing (§276's table).
func Held(step string) bool { return step == seatdef.StepCompose || step == seatdef.StepPlace }
// Wait is one wait as kept: Module's entry on Node waits for Part's version Fp to run on PartNode, in Step.
type Wait struct {
V int `json:"v"`
Step string `json:"step"`
Node string `json:"node"`
Module string `json:"module"`
Part string `json:"part"`
PartNode string `json:"part-node"`
Fp string `json:"fp"`
Entry string `json:"entry"`
Since time.Time `json:"since"`
ClockFrom time.Time `json:"clock-from"`
Bound string `json:"bound"`
// Released is a release by hand: the entry moves on with the part's old version; EndSaid says wait-ended was said for it.
// Order is which of §298's edges the wait stands on; empty for an acting wait.
Order string `json:"order,omitempty"`
Released bool `json:"released,omitempty"`
EndSaid bool `json:"end-said,omitempty"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
}
// Key is the key of a wait in the state.
func Key(node, module, part string) string { return Prefix + node + "." + module + "." + part }
// OrderKey is the key of an order wait in the state.
func OrderKey(node, module, part string) string {
return OrderPrefix + node + "." + module + "." + part
}
// IsOrder says a kept value is an order wait's record.
func IsOrder(key string) bool { return strings.HasPrefix(key, OrderPrefix) }
// Is says a kept value is an acting wait's record.
func Is(key string) bool { return strings.HasPrefix(key, Prefix) }
// Encode is w as JSON.
func Encode(w Wait) ([]byte, error) { w.V = 1; return json.Marshal(w) }
// Decode reads a kept wait.
func Decode(raw []byte) (Wait, error) {
var w Wait
if err := json.Unmarshal(raw, &w); err != nil {
return w, err
}
if w.V != 1 || w.Step == "" || w.Node == "" || w.Module == "" || w.Part == "" {
return w, fmt.Errorf("acting: %q is not a kept acting wait", raw)
}
return w, nil
}
// BoundOf is the bound of the edge's wait: its rule's, else its step's default (seatdef.ActingRule.Wait).
func BoundOf(e meshwants.Edge) time.Duration {
return seatdef.ActingRule{Step: e.Step, Bound: e.Bound}.Wait()
}
// Pending are the acting edges of the assigner's steps whose part's version does not run yet: runs says the edge's part runs
// its version on the edge's node.
func Pending(edges []meshwants.Edge, runs func(e meshwants.Edge) bool) []meshwants.Edge {
var out []meshwants.Edge
for _, e := range edges {
if e.Kind == "acting" && Held(e.Step) && !runs(e) {
out = append(out, e)
}
}
return out
}
// Body is the wait as the seat mesh-wants says it.
func Body(w Wait, owner string) meshwants.Wait {
since, from := w.Since.UTC(), w.ClockFrom.UTC()
return meshwants.Wait{Step: w.Step, Owner: owner, Node: w.Node, Module: w.Module, Part: w.Part, Fp: w.Fp, Since: since, ClockFrom: &from, Bound: w.Bound,
Words: words(w)}
}
func words(w Wait) string {
switch w.Order {
case OrderBackup:
return fmt.Sprintf("%s on %s moves its kept data across a form change: it waits in %s for %s at %s to run on %s, which the restore point needs", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
case OrderEngineLast:
return fmt.Sprintf("%s on %s goes back last on its node: it waits in %s for %s at %s to run on %s, restored", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
case OrderTakenIn:
return fmt.Sprintf("%s on %s goes back only once the indexer took in the trunk's head: it waits in %s for source revision %s to be taken in", w.Module, w.Node, w.Step, w.Fp)
}
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", w.Module, w.Node, w.Step, w.Part, short(w.Fp), w.PartNode)
}
func short(fp string) string {
if len(fp) > 19 {
return fp[:19]
}
return fp
}
@@ -1,69 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
func assignerAsk(asking int, a oc.Ask) asks.Ask {
raw, err := json.Marshal(a)
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
}
return asks.Ask{Asking: asking, Ask: raw}
}
// AskOf is the router's ask an asks-state value carries.
func AskOf(a asks.Ask) (oc.Ask, error) {
var ask oc.Ask
if err := json.Unmarshal(a.Ask, &ask); err != nil {
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
}
return ask, nil
}
// EncodeAsk is asks.EncodeAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
// the ask's id is meshdeliveryclient.AskingID(key, asking), and it offers at least one option.
func EncodeAsk(key string, a asks.Ask) ([]byte, error) {
if _, err := checkedAsk(key, a); err != nil {
return nil, err
}
return asks.EncodeAsk(key, a)
}
func checkedAsk(key string, a asks.Ask) (oc.Ask, error) {
ask, err := occ.Carried(a.Ask, meshdeliveryclient.AskingID(key, a.Asking))
if err != nil {
return oc.Ask{}, fmt.Errorf("assign: %w", err)
}
return ask, nil
}
// DecodeAsk reads the ask at key (asks.DecodeAsk) with the router's ask it carries.
func DecodeAsk(key string, data []byte) (asks.Ask, oc.Ask, error) {
a, err := asks.DecodeAsk(key, data)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
ask, err := checkedAsk(key, a)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
return a, ask, nil
}
// ExpiresAt is when the ask ends unanswered; the zero time for an ask that names none.
func ExpiresAt(a oc.Ask) time.Time {
t, err := occ.Time(a.Expires)
if err != nil {
return time.Time{}
}
return t
}
@@ -2,40 +2,34 @@ package assign
import (
"fmt"
"mesh-delivery-assigner/internal/assignment"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Options: []oc.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: occ.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the version stays held",
About: key,
})
}}
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
@@ -45,148 +39,72 @@ const (
)
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. asking is the writing's
// number (AskingSeconds): an ask taken away as stale and written again is another asking under an id of its own, since the
// router drops a second ask under an id it keeps and the teller took the first back (issue 1966). Restore sets the
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []oc.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: occ.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module stays as it is on the node",
About: key,
})
}
// The options of the outside ask (design 47 G §706).
const (
OptionGoOn = "go-on"
OptionFailed = "failed"
)
// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
Options: []oc.Option{
{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the delivery keeps waiting",
About: about,
})
}
// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
Severity: "warning",
Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
}}
}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
key := assignment.Module() + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
c := oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = occ.NeedsYou + c.Needs + ". " + what
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = occ.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) oc.Condition {
key := assignment.Module() + ".failed-alone." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", occ.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) oc.Condition {
key := assignment.Module() + ".neither." + node + "." + module
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
key := delivery.AssignerModule + ".neither." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared,
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = string([]rune(headline)[:occ.HeadlineLength])
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return oc.Condition{Key: key, Severity: occ.SeverityUrgent, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", occ.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) oc.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) oc.Condition {
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
@@ -196,91 +114,37 @@ func LiftedCondition(module, fp string) oc.Condition {
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = mwc.BackupAgain
OptionBackupSkip = mwc.BackupSkip
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
)
// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
const (
OptionRevertHold = "hold"
OptionRevertLive = mwc.RevertLive
)
// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
const KindRevertHeld = "revert-held"
// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
// the asking, 0 the first, as the backup ask's.
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
module, node, short(fp), kept, form, words, module),
Options: []oc.Option{
{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the module keeps running its newer build",
About: key,
})
}
// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
func RevertAnswerWords(option string, asking int) string {
return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
}
// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
// operator's one message is the ask.
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
module, reason, kept, form, module),
Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
}
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
// asked again, for the rewritten key carried the id the router already kept).
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + need.Words,
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
Options: []oc.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the node keeps what it runs",
About: key,
})
}}
}
// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
@@ -293,12 +157,8 @@ var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a asks.Ask) string {
ask, err := AskOf(a)
if err != nil {
return ""
}
if m := backupDigest.FindStringSubmatch(ask.Explanation); m != nil {
func BackupAskDigest(a delivery.AssignerAsk) string {
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
return m[1]
}
return ""
@@ -313,34 +173,34 @@ func BackupAnswerWords(option string, asking int) string {
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Options: []oc.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module keeps running what it has",
About: key,
})
}}
}
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
needs := "answer whether to back up again or go on without a backup"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}
@@ -1,366 +1,136 @@
package assign
import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/puts"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
// its key, of a kind the view may answer.
func TestTheHoldAskIsValid(t *testing.T) {
t.Run("hold", theHoldAskIsValid)
t.Run("restore", theRestoreAskIsValid)
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
}
// Issue 1966, note 151: an ask the teller says ended at the operator channel without an answer (its event ask-ended) is
// written again under a new asking, an id of its own and a new expiry, one row per kind: the rules' asks are; the
// authorising ask of a proposal (ADR 0567 §5, §11) and a put's record ask (ADR 0546) are not, nor an asking the key no longer
// holds, an end asking again would not change, or a body that does not read.
func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
now := asked.Add(8 * 24 * time.Hour)
digest := "sha256:" + strings.Repeat("e", 64)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
assignmentsKey, _ := asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")
need := RecordNeed{Node: "novox", Digest: digest, Judgement: JudgedMovesData}
record := RecordAsk(recordKey, "novox", "nats", fNew, need, 0, asked) // written before #195: no asking
put := puts.Put{FP: fNew, Why: "w", At: asked, Step: puts.Person, Ask: recordKey}
byPut := RecordAsk(recordKey, "novox", "nats", fNew, need, PutAsking(put), asked)
other := puts.Put{FP: fNew, Why: "w", At: asked.Add(time.Hour), Step: puts.Person, Ask: recordKey}
assignments := assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(assignmentsKey, AskingSeconds(asked)), Headline: "h",
Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(occ.ApproveLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: OptionYes, Label: "Yes", Does: "d", Level: occ.Approve}}})
for _, c := range []struct {
name string
key string
a *asks.Ask // the ask the key holds; nil for none
ended string // the id of the asking that ended; "" for the one the key holds
outcome string
put *puts.Put
why string // "" when it is written again
}{
{name: "hold", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeExpired},
{name: "restore", key: restoreKey, a: ptr(RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked), asked)), outcome: occ.OutcomeCancelled},
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, digest, "w", 0, asked)), outcome: occ.OutcomeCancelled},
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
{name: "record", key: recordKey, a: &record, outcome: occ.OutcomeCancelled},
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
{name: "another asking stands", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(now), now)),
ended: meshdeliveryclient.AskingID(holdKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
{name: "taken away", key: holdKey, ended: "hold-x", outcome: occ.OutcomeExpired, why: "taken away"},
{name: "refused", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
{name: "replaced", key: holdKey, a: ptr(HoldAsk(holdKey, "nats", fNew, "novox", "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
} {
t.Run(c.name, func(t *testing.T) {
var current []byte
ended := c.ended
if c.a != nil {
var err error
if current, err = EncodeAsk(c.key, *c.a); err != nil {
t.Fatal(err)
}
if ended == "" {
ended = mustAsk(t, *c.a).ID
}
}
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: c.key, Ask: ended, Outcome: c.outcome})
if err != nil {
t.Fatal(err)
}
e, err := tellerevents.DecodeEnded(body)
if err != nil {
t.Fatal(err)
}
value, why := AskedAgain(e, current, c.put, now)
if c.why != "" {
if value != nil || !strings.Contains(why, c.why) {
t.Fatalf("AskedAgain = %s, %q; want nothing written, %q", value, why, c.why)
}
return
}
if why != "" {
t.Fatalf("AskedAgain says %q, want the ask written again", why)
}
a, ask, err := DecodeAsk(c.key, value)
if err != nil {
t.Fatal(err)
}
if a.Asking <= c.a.Asking || ask.ID != meshdeliveryclient.AskingID(c.key, a.Asking) || ask.ID == ended {
t.Errorf("asking %d under %s, want a new asking above %d under an id of its own, not %s", a.Asking, ask.ID, c.a.Asking, ended)
}
if err := occ.Check(ask, now); err != nil || !ExpiresAt(ask).Equal(now.Add(AskLasts).Truncate(time.Second)) {
t.Errorf("the router refuses the ask written again (%v), or it lapses at %s, want %s", err, ExpiresAt(ask), now.Add(AskLasts))
}
})
}
for _, raw := range []string{`{"v":1,"key":"k","ask":"a"}`, `{"v":1,"key":"k","ask":"a","outcome":"expired","more":1}`} {
if _, err := tellerevents.DecodeEnded([]byte(raw)); err == nil {
t.Errorf("DecodeEnded(%s) takes a body that is not an ended ask", raw)
}
}
}
// Issue 1966 note 155, ADR 0668 §4: an ended ask the teller said before the assigner's consumer existed is not delivered to
// it, so at bind the assigner reads the operator-channel seat's asks: the query reaches back to the earliest held asking,
// and of the answer only the held askings that lapsed or were cancelled are taken, as the teller's event says them; an
// answered or another asker's ask, and one the key no longer holds, are not.
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
holdKey, _ := asks.HoldAskKey("nats", fNew)
restoreKey, _ := asks.RestoreAskKey("novox", "nats", fNew)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
hold := HoldAsk(holdKey, "nats", fNew, "novox", "w", AskingSeconds(asked), asked)
restore := RestoreAsk(restoreKey, "novox", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
backup := BackupAsk(backupKey, "novox", "nats", fNew, "sha256:"+strings.Repeat("e", 64), "w", 0, asked.Add(-time.Hour)) // no asking
held := map[string][]byte{}
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
v, err := EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
}
held[key] = v
}
if _, ok := ChannelAsksQuery(nil); ok {
t.Error("with no ask held the channel is asked")
}
q, ok := ChannelAsksQuery(held)
if !ok || q.State != "ended" || q.Bound != ChannelAsksBound || q.Since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) {
t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339))
}
at := func(key string, a asks.Ask, state string) string {
return fmt.Sprintf(`{"key":"t.%s","state":%q}`, mustAsk(t, a).ID, state)
}
answer := `{"asks":[` + strings.Join([]string{at(holdKey, hold, occ.OutcomeExpired), at(restoreKey, restore, occ.OutcomeChosen),
at(backupKey, backup, occ.OutcomeCancelled), `{"key":"t.hold-gone","state":"expired"}`, `{"key":"open","state":"open"}`}, ",") + `],"cut":"c"}`
ended, cut, err := EndedAtChannel(held, []byte(answer))
want := []tellerevents.Ended{{Key: backupKey, Ask: mustAsk(t, backup).ID, Outcome: occ.OutcomeCancelled},
{Key: holdKey, Ask: mustAsk(t, hold).ID, Outcome: occ.OutcomeExpired}}
slices.SortFunc(want, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
if err != nil || cut != "c" || !slices.Equal(ended, want) {
t.Errorf("EndedAtChannel = %+v, %q, %v; want %+v and the answer's cut", ended, cut, err, want)
}
if _, _, err := EndedAtChannel(held, []byte(`[1]`)); err == nil {
t.Error("an answer that does not read is taken")
}
}
func ptr(a asks.Ask) *asks.Ask { return &a }
func theHoldAskIsValid(t *testing.T) {
key, err := asks.HoldAskKey("mail", fNew)
key, err := delivery.HoldAskKey("mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := EncodeAsk(key, again); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
good := HoldAsk(key, "mail", fNew, "ace", "x", 0, time.Unix(100, 0))
other := good
other.Asking = 3 // its id is the asking 0's
empty := good
ask := mustAsk(t, good)
ask.Options = nil
empty.Ask = assignerAsk(0, ask).Ask
for name, a := range map[string]asks.Ask{"another id": other, "no option": empty, "no ask": {}} {
if _, err := EncodeAsk(key, a); err == nil {
t.Errorf("%s: written", name)
}
}
})
}
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func theRestoreAskIsValid(t *testing.T) {
key, err := asks.RestoreAskKey("ace", "mail", fNew)
func TestTheRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", 2, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range mustAsk(t, a).Options {
for _, o := range a.Ask.Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(mustAsk(t, a).Explanation, "connection refused") || !strings.Contains(mustAsk(t, a).Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", mustAsk(t, a).Explanation)
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
}
}
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
func TestTheAssignersConditions(t *testing.T) {
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
}
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != occ.StateCleared || got.Key != c.Key {
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
// findings bucket cannot be written to at all.
func saidConditionsAreFindings(t *testing.T) {
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
for _, c := range []struct {
raised, cleared oc.Condition
key string
}{
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]mcseat.Finding{}, []string{}, 0
f := Findings{Put: func(f mcseat.Finding) error {
puts[views.FindingKey(f.Scope, f.ID, f.Kind)] = f
return nil
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
say := f.Say(func(oc.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
t.Fatalf("%s: %v", c.key, err)
}
got, ok := puts[c.key]
if !ok || got.Severity != c.raised.Severity || got.Headline != c.raised.Headline || got.Observer != assignment.Module() ||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
}
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
}
if err := (Findings{}).Say(func(oc.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
}
}
if _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
t.Error("a condition not the assigner's is a finding")
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := asks.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, 2, now)
bkey, err := asks.BackupAskKey("ace", "mail", fNew)
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
v := RevertAsk(vkey, "ace", "mail", fNew, "records", "3", "restore point missing: none of form 3", 0, now)
for key, a := range map[string]asks.Ask{rkey: r, bkey: b, vkey: v} {
if err := occ.Check(mustAsk(t, a), now); err != nil {
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
if err := a.Ask.Check(now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", mustAsk(t, r))
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", r.Ask)
}
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
t.Errorf("the backup ask = %+v", mustAsk(t, b))
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
t.Errorf("the backup ask = %+v", b.Ask)
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
@@ -372,68 +142,45 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
t.Errorf("two answers of the same option are one want value")
}
// design 47 G §710: the revert ask names the build, the kept data, its form and the node's words, and offers hold and live;
// its finding is delivery.<node>.<module>.revert-held, a warning that needs nothing, the ask being the one message.
if va := mustAsk(t, v); va.Headline != "The revert of mail on ace has no restore point" || va.Options[0].ID != OptionRevertHold ||
va.Options[1].ID != mwc.RevertLive || len(va.Options) != 2 || va.OnExpiry != "the module keeps running its newer build" ||
!strings.Contains(va.Explanation, "records") || !strings.Contains(va.Explanation, "form 3") || !strings.Contains(va.Explanation, short(fNew)) ||
!strings.Contains(va.Explanation, "Started on the live data, the older build reads data written in the newer form.") {
t.Errorf("the revert ask = %+v", va)
}
if f := RevertHeldFinding("ace", "mail", "records", "3", "restore point missing").Finding(now); views.FindingKey(f.Scope, f.ID, f.Kind) != "delivery.ace.mail.revert-held" ||
f.Severity != "warning" || f.Needs != "Nothing for you to do." ||
f.Summary != "The revert of mail on ace is held: restore point missing (records, form 3); the ask about it waits for your answer" {
t.Errorf("the revert finding = %+v", f)
}
if RevertAnswerWords(OptionRevertLive, 1) != `the operator answered "live" to the revert ask (asking 2)` {
t.Errorf("the words of a live answer: %q", RevertAnswerWords(OptionRevertLive, 1))
}
c := BackupCondition("ace", "mail", fNew, "no space left", true)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
t.Fatalf("bus %v: %v", bus, err)
}
if err := occ.Check(mustAsk(t, a), now); err != nil {
if err := a.Ask.Check(now); err != nil {
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
}
if _, err := EncodeAsk(rkey, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := viewstate.ViewAnswerFits(viewstate.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
yes := mustAsk(t, a).Options[0]
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
// acknowledge), as it did through the sdk before.
if viewed == nil {
if o, ok := occ.OptionOf(mustAsk(t, a), OptionYes); ok && o.Level != "" && o.Level != occ.Acknowledge {
viewed = fmt.Errorf("only a channel that proves who answered takes the option %s", o.ID)
}
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
yes := a.Ask.Options[0]
switch {
case !strings.Contains(mustAsk(t, a).Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, mustAsk(t, a).Explanation)
case bus && (yes.Level != occ.Approve || yes.Binds == "" || viewed == nil || expiresAt(t, a).After(now.Add(occ.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, expiresAt(t, a))
case !bus && (yes.Level != occ.Acknowledge || viewed != nil):
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
}
later := put
later.At = now.Add(time.Minute)
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
if mustAsk(t, b).ID == mustAsk(t, a).ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, mustAsk(t, b).ID, mustAsk(t, a).ID)
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dgst)); err != nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
t.Errorf("bus: the yes performs another composition's act")
}
}
@@ -442,30 +189,15 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := occ.EncodeCondition(c); err != nil {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != occ.StateRaised || c.Severity != occ.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != occ.StateCleared || got.Key != c.Key {
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// mustAsk is the router's ask an asks-state value carries.
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
t.Helper()
ask, err := AskOf(a)
if err != nil {
t.Fatal(err)
}
return ask
}
func expiresAt(t *testing.T, a asks.Ask) time.Time {
t.Helper()
return ExpiresAt(mustAsk(t, a))
}
@@ -1,23 +0,0 @@
package assign
import (
"mesh-delivery-assigner/internal/assignment"
"strings"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// SeatWaitsCondition is the assigner's word on the seat mesh-module-assigner while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) oc.Condition {
key := assignment.Module() + ".seat-states"
subject := oc.ConditionSubject{Scope: "module", ID: assignment.Module()}
if len(unbound) == 0 {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, Resolved: "the seat mesh-module-assigner holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: occ.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -0,0 +1,282 @@
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
// and writes nothing itself: the program hands it what it watched and writes what it answers.
package assign
import (
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
const SDKRepository = "novox/mesh-sdk"
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
"node-engine's validator would refuse"
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
// why, and each node's open assignments.
type Mesh struct {
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Assigned map[string][]string
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
// member (empty when none), and how many modules it rebuilds.
type Change struct {
Group, Heads string
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Removed []string
Twice map[string][]string
SDKHead string
Rebuilt int
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine.
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
moved[name] = true
}
for name := range c.Unreadable {
moved[name] = true
}
for _, name := range c.Removed {
moved[name] = true
}
provided, required := map[string]bool{}, map[string]bool{}
for name := range moved {
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
for _, o := range man.Provides {
provided[o.Name] = true
}
}
for _, r := range c.Manifests[name].Requires {
required[r] = true
}
}
var out []string
for node, modules := range m.Assigned {
for _, mod := range modules {
man := m.Manifests[mod]
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
in := compose.Input{Catalogue: catalog}
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
for _, mod := range assigned[n] {
for _, cl := range catalog[mod].Claims {
scope := cl.Scope
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
scope = seat.Scope
}
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
}
}
}
return in
}
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
var fails []string
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Unreadable) {
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
machines := Touched(m, c)
v.Machines = machines
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
}
trunk := input(m.Manifests, m.Assigned)
changed := map[string]catalogue.Manifest{}
for name, man := range m.Manifests {
changed[name] = man
}
for name, man := range c.Manifests {
changed[name] = man
}
for _, name := range c.Removed {
delete(changed, name)
}
after := input(changed, m.Assigned)
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
switch {
case now == nil:
case before != nil:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
}
}
if c.SDKHead != "" {
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
}
return finish(v, m, c, machines, fails)
}
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
if c.Rebuilt > delivery.CompositionRebuildWarning {
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
}
v.Warnings = append(v.Warnings, NotJudged)
if could := CouldRun(m, c, machines); len(could) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
}
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
}
if len(machines) == 0 {
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
}
if len(fails) > 0 {
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
return v
}
v.State = delivery.CompositionPass
return v
}
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
// the facts snapshot), so each is named as not judged.
func CouldRun(m Mesh, c Change, composed []string) []string {
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for name := range c.Manifests {
if !slices.Contains(modules, name) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
func grantsOf(man catalogue.Manifest) string {
var offers []string
for _, o := range man.Provides {
offers = append(offers, o.Name)
}
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
}
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
func GrantsAlone(m Mesh, c Change, composed []string) []string {
provisions := map[string]bool{}
altered := false
for name, man := range c.Manifests {
before, had := m.Manifests[name]
if had && grantsOf(before) == grantsOf(man) {
continue
}
altered = true
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
provisions[r] = true
}
}
if !altered {
return nil
}
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for _, mod := range modules {
man := m.Manifests[mod]
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -0,0 +1,141 @@
package assign
import (
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func meshOf() Mesh {
return Mesh{
Manifests: map[string]catalogue.Manifest{
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
"web": {Module: "web", Version: "1"},
},
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
}
}
func hasWarning(v delivery.CompositionVerdict, part string) bool {
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
}
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
// the machine and the mesh-sdk it composed with, and says what it did not judge.
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
v := Compose(meshOf(), c, "v0.1.14-x", 0)
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
}
if !hasWarning(v, "issue 1461") {
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
}
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
}
}
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
// consumer's machine is touched when its provider changes.
func TestAMachineTheChangeBreaksFails(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
}
}
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
m := meshOf()
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
v := Compose(m, c, "sdk", 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
}
}
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
func TestUnreadableTwiceAndWide(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
t.Fatalf("composition = %+v", v)
}
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
v = Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
t.Fatalf("composition = %+v", v)
}
}
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
v := Compose(meshOf(), c, "v0.1.14-own", 0)
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
}
c.SDKHead = ""
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
}
}
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
// assigned nowhere touches none, and the verdict says so.
func TestTouched(t *testing.T) {
m := meshOf()
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
t.Errorf("touched by store = %v", got)
}
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
t.Errorf("touched by store with a consumer on g14 = %v", got)
}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
t.Errorf("an added module assigned nowhere: %+v", v)
}
}
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
// names none of the second.
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
m := meshOf()
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
m.Assigned["novox"] = []string{"bus"}
m.Assigned["shanks"] = []string{"db"}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
v := Compose(m, c, "sdk", 0)
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
}
if !hasWarning(v, "whose grants alone the change alters: novox") {
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
}
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
}
}
@@ -1,45 +0,0 @@
package assign
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
// judgement); neither and unchanged are the assigner's own.
type Judgement = string
// The judgements of a move.
const (
// JudgedRecreates is a move that recreates a container.
JudgedRecreates = nec.JudgedRecreates
// JudgedMovesData is a move that gives the module's data another place.
JudgedMovesData = nec.JudgedMovesData
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
JudgedNeither = "neither"
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
JudgedCannot = nec.JudgedCannot
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
// changes nothing on the node, so it needs no person's yes.
JudgedUnchanged = "unchanged"
)
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
// the module). Words say why, and are required for a move that cannot be judged.
//
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
type ComposedDigest struct {
V int `json:"v"`
Digest string `json:"digest"`
Judgement Judgement `json:"judgement"`
Against string `json:"against,omitempty"`
Words string `json:"words,omitempty"`
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
// record hold (ADR 0539 clause 7).
UserListOnly bool `json:"user-list-only,omitempty"`
Policy string `json:"policy,omitempty"`
PolicyFrom string `json:"policy-from,omitempty"`
PolicyWhy string `json:"policy-why,omitempty"`
}
@@ -0,0 +1,241 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
const (
CaseTrunk = "trunk"
CaseChange = "change"
)
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
const SDKDir = "go"
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
// them, the node-engine applies them. Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
for name, digest := range m.Digests {
trunk[name] = composer.Manifest{Digest: digest}
change[name] = composer.Manifest{Digest: digest}
}
for name, digest := range c.Digests {
change[name] = composer.Manifest{Digest: digest}
}
for _, name := range c.Removed {
delete(change, name)
}
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
HoldingsFromClaims: true, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
changed := map[string]composer.Manifest{}
for name, digest := range c.Digests {
changed[name] = composer.Manifest{Digest: digest}
}
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
}
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
raw, err := json.Marshal(req)
if err != nil {
return delivery.ComposeJob{}, err
}
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
Package: composer.Package, Request: raw, Reader: reader}, nil
}
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
// composer derives the seats held from the claims, as input does).
func plainInput(assigned map[string][]string) compose.Input {
var in compose.Input
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
}
return in
}
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
type Ran struct {
State string
Reason string
Verdict *composer.Verdict
}
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
// failure, named.
func RanOf(step delivery.Step) Ran {
r := Ran{State: step.State, Reason: step.Reason}
if step.State != delivery.StepPassed {
return r
}
var a delivery.ComposeAnswer
var v composer.Verdict
if err := json.Unmarshal(step.Result, &a); err != nil {
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
}
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
}
r.Verdict = &v
return r
}
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
if v == nil {
return composer.CaseVerdict{}, false
}
for _, c := range v.Cases {
if c.Name == name {
return c, true
}
}
return composer.CaseVerdict{}, false
}
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
// the group holds one in its repository, else a build its node runs.
type Reader struct {
Module, Commit string
Member bool
Ran Ran
}
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
// said and judged nothing.
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
machines := Touched(m, c)
v.Machines = machines
var fails []string
if head.State != delivery.StepPassed {
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
v.State = delivery.CompositionFail
return v
}
if head.Verdict.SDK != "" {
v.SDK = head.Verdict.SDK
}
trunk, okT := caseOf(head.Verdict, CaseTrunk)
change, okC := caseOf(head.Verdict, CaseChange)
if !okT || !okC {
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
return v
}
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Digests) {
if r := change.Manifests[name]; !r.Accepted {
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
}
}
for _, name := range sortedKeys(trunk.Manifests) {
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
continue
}
if _, before := m.Unreadable[name]; before {
continue // already unreadable on the trunk: not the group's doing
}
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
before, now := trunk.Machines[node], change.Machines[node]
switch {
case now.Composes:
case !before.Composes:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
}
}
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
return finish(v, m, c, machines, fails)
}
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
// from before issue 1621): Go's words for a package its module does not hold.
func NoComposer(reason string) bool {
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
if strings.Contains(reason, words) {
return true
}
}
return false
}
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
for _, r := range readers {
at := r.Module + " at " + r.Commit
if r.Member {
at = r.Module + "'s member head " + r.Commit
}
if r.Ran.State != delivery.StepPassed {
if NoComposer(r.Ran.Reason) {
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
"composer command (before issue 1621)", at))
continue
}
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
continue
}
read, ok := caseOf(r.Ran.Verdict, CaseChange)
if !ok {
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
continue
}
for _, name := range sortedKeys(c.Digests) {
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
continue
}
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
}
}
return fails, warnings
}
@@ -0,0 +1,184 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
const sdkHead = "1111111111111111111111111111111111111111"
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
var raws = map[string]string{
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
"d-web": `{"module": "web", "version": "1"}`,
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
}
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
func digestedMesh() Mesh {
m := meshOf()
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
return m
}
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
func webShares(members map[string]string) Change {
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
}
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
t.Helper()
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if err := composer.Fill(&req, func(d string) ([]byte, error) {
if r, ok := read[d]; ok {
return []byte(r), nil
}
return []byte(raws[d]), nil
}); err != nil {
t.Fatal(err)
}
v, err := composer.Run(req)
if err != nil {
t.Fatal(err)
}
v.SDK = "(devel) " + job.Fetch["commit"].(string)
vr, _ := json.Marshal(v)
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
}
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
// change by digest, never the manifests' bytes, composing the machines the change touches.
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
c := webShares(nil)
job, err := HeadJob(digestedMesh(), c)
if err != nil {
t.Fatal(err)
}
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
t.Fatalf("job = %+v", job)
}
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
At: testTime, Compose: &job}
if _, err := delivery.EncodeStepAsk(ask); err != nil {
t.Fatalf("the ask is refused by its layout: %v", err)
}
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
t.Fatalf("cases = %+v", req.Cases)
}
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
}
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
v := FromComposer(digestedMesh(), c, head, nil, 0)
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
t.Fatalf("with the head's composer: %+v", v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
}
}
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
// the SDK head (its composer reads what the head reads) it passes.
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
running := "3333333333333333333333333333333333333333"
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
if err != nil {
t.Fatal(err)
}
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
t.Fatalf("reader job = %+v", rjob)
}
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
}
member := "2222222222222222222222222222222222222222"
pinned := run(t, rjob, asTheHeadReads)
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
if v.State != delivery.CompositionPass {
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
}
}
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
// failed for another cause fails the group, loud.
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
t.Fatalf("a reader with no composer: %+v", v)
}
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
t.Fatalf("a reader whose composer failed: %+v", v)
}
}
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
c := webShares(nil)
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
t.Fatalf("a head composer that did not run: %+v", v)
}
job, _ := HeadJob(digestedMesh(), c)
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
}
}
@@ -1,139 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/puts"
)
// AskLasts is how long an ask of the assigner's rules stands at the operator channel before it lapses unanswered: the hold,
// restore, record, backup, outside and revert asks.
const AskLasts = 7 * 24 * time.Hour
// writtenAgainOnEnd is, per kind of the assigner's asks, whether an asking that ended at the channel without an answer while
// its key stands is written again under a new asking (issue 1966, note 151), or why not. A rule's ask waits for a person
// as long as its key stands, so it is asked again; the authorising ask of a proposal ends the proposal when it lapses
// (ADR 0567 §5) and is asked again by the seat's verb `again` (§11).
var writtenAgainOnEnd = map[string]string{
asks.AskKindHold: "",
asks.AskKindRestore: "",
asks.AskKindBackup: "",
asks.AskKindOutside: "", // raised again at once while the wait stands (design 47 G §706); the wait's end takes the key away
asks.AskKindRecord: "",
asks.AskKindRevert: "", // raised again at once while the hold stands (design 47 G §710); one answered hold is not asked again
asks.AskKindAssignments: "a proposal's authorising ask that lapsed ends the proposal (ADR 0567 §5); the verb again asks it again (§11)",
}
// AskedAgain is the ask at key written again after the teller said its asking ended at the channel without an answer (the
// teller's event ask-ended): the value to write by compare-and-set over current, or why nothing is written. Nothing is written
// for an outcome other than a lapse or a cancellation, for a key that holds another asking now (an answer to it, or a newer
// asking, came since), for a kind its own rule ends (writtenAgainOnEnd), and for a record ask a put asked (put, nil for none:
// the put at the record ask's node and module), since its put is refused when it lapses (ADR 0546). The new asking is
// AskingSeconds(now), above the old one, under an id of its own, standing AskLasts from now.
func AskedAgain(e tellerevents.Ended, current []byte, put *puts.Put, now time.Time) ([]byte, string) {
if e.Outcome != occ.OutcomeExpired && e.Outcome != occ.OutcomeCancelled {
return nil, fmt.Sprintf("it ended %s, which asking again would not change", e.Outcome)
}
k, err := asks.ParseAnyAskKey(e.Key)
if err != nil {
return nil, err.Error()
}
why, known := writtenAgainOnEnd[k.Kind]
if !known {
return nil, "the assigner asks no " + k.Kind + " ask"
}
if why != "" {
return nil, why
}
if current == nil {
return nil, "the key is taken away: nothing is asked"
}
a, ask, err := DecodeAsk(e.Key, current)
if err != nil {
return nil, err.Error()
}
if ask.ID != e.Ask {
return nil, "the key holds the asking " + ask.ID + " now, not the one that ended"
}
if k.Kind == asks.AskKindRecord && put != nil && put.Ask == e.Key && PutAsking(*put) == a.Asking {
return nil, "a put asked it, and its put is refused when it lapses (ADR 0546)"
}
asking := max(AskingSeconds(now), a.Asking+1)
ask.ID, ask.Expires = meshdeliveryclient.AskingID(e.Key, asking), occ.FormatTime(now.Add(AskLasts))
value, err := EncodeAsk(e.Key, assignerAsk(asking, ask))
if err != nil {
return nil, err.Error()
}
return value, ""
}
// ChannelAsksBound is the most bytes of asks the assigner asks the operator-channel seat's verb asks for at its consumer's
// bind: every ended asking of its held asks fits.
const ChannelAsksBound = 1 << 20
// ChannelAsksQuery is the arguments of the operator-channel seat's verb asks that answer, at the assigner's consumer's bind,
// every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks
// opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false
// when no held ask reads, and then nothing is asked.
func ChannelAsksQuery(held map[string][]byte) (query oc.AsksInput, ok bool) {
var since time.Time
for key, value := range held {
a, ask, err := DecodeAsk(key, value)
if err != nil {
continue
}
opened := ExpiresAt(ask).Add(-AskLasts)
if a.Asking >= 2 {
if at := time.Unix(int64(a.Asking), 0); at.Before(opened) {
opened = at
}
}
if !ok || opened.Before(since) {
since, ok = opened, true
}
}
if !ok {
return oc.AsksInput{}, false
}
return oc.AsksInput{State: "ended", Since: since.Add(-time.Minute).UTC().Format(time.RFC3339), Bound: ChannelAsksBound}, true
}
// EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there
// expired or cancelled, as the teller says it in its event ask-ended, so the assigner writes it again by the rules of that
// event (AskedAgain), with its compare-and-set: an ended ask said before the assigner's consumer existed is not delivered
// to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read
// is the error.
func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) {
var got oc.AsksAnswer // the seat's declared answer: each ask's key and its state, and the cut
if err := json.Unmarshal(answer, &got); err != nil {
return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err)
}
byID := map[string]string{}
for key, value := range held {
if _, ask, err := DecodeAsk(key, value); err == nil {
byID[ask.ID] = key
}
}
for _, v := range got.Asks {
if v.State != occ.OutcomeExpired && v.State != occ.OutcomeCancelled {
continue
}
_, id, ok := oc.ParseAsksByAskerIDKey(v.Key)
if key, held := byID[id]; ok && held {
ended = append(ended, tellerevents.Ended{Key: key, Ask: id, Outcome: v.State})
}
}
slices.SortFunc(ended, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
return ended, got.Cut, nil
}
@@ -1,171 +0,0 @@
package assign
import (
"errors"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
)
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair
// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its
// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router
// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the
// assigner no longer says it straight (never both).
// FindingsBucket is the assigner's findings.
// FindingsState is the assigner's state of its findings, and FindingsBucket its bucket.
const FindingsState = "findings"
func FindingsBucket() string { return assignment.Module() + "_" + FindingsState }
// KeyID is an id as the parts of a key: each dot-separated part made safe for a bus key, as the key of a condition is
// (mcc.Key), so a finding's key keeps the shape its condition had.
func KeyID(id string) string {
key := mcc.Key("x", id, "x")
return strings.TrimSuffix(strings.TrimPrefix(key, "x."), ".x")
}
// Observed is what the assigner observes wrong, before it is the seat mesh-conditions' generated finding.
type Observed struct {
Scope, ID, Kind, Machine string
Also []string
Severity string
Summary, Said string
Headline, Explanation string
Needs, Source string
Actions []mcseat.Action
}
// Finding is o as the seat's generated finding at at (design 47 G §686, A-22): its id made safe as a key's part, the resolved
// words not a field of it, its observer the assigner; the writer completes its `since`.
func (o Observed) Finding(at time.Time) mcseat.Finding {
at = at.UTC()
id := KeyID(o.ID)
said := o.Said
if said == "" {
said = o.Summary
}
sev := o.Severity
if sev == "" {
sev = mcseat.FindingSeverityWarning
}
return mcseat.Finding{Scope: o.Scope, ID: id, Kind: o.Kind, Severity: sev, Summary: o.Summary, Headline: o.Headline,
Explanation: o.Explanation, Needs: o.Needs, Actions: o.Actions, Source: o.Source, Observer: assignment.Module(), At: at,
Evidence: []mcseat.Evidence{{At: at, Said: said}},
Subject: mcseat.Subject{Scope: o.Scope, ID: id, Machine: o.Machine, Also: o.Also}}
}
// CallUnanswered is the observation of a kept call to callee (a seat verb's or a tool's key) about id (what the assigner waits
// on: a node, a key), unanswered since since, its last fault said. Its key is `call.<id>.call-unanswered`.
func CallUnanswered(id, callee, machine, fault, source string, since time.Time) Observed {
return Observed{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Machine: machine, Severity: mcseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the call %s about %s is unanswered since %s", callee, id, since.UTC().Format(time.RFC3339)),
Said: fault, Headline: "A call whose answer is kept is not answered",
Explanation: fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has "+
"not been answered since %s. Its last fault: %s.", callee, since.UTC().Format(time.RFC3339), fault),
Source: source}
}
// FindingOf is a said condition as a finding: from the condition's own key `mesh-delivery-assigner.<kind>.<id…>` (the
// subject's id when the key names none), its words unchanged but the resolved ones. raised is false for a clear, whose
// finding names only its scope, id and kind.
func FindingOf(c oc.Condition, at time.Time) (f mcseat.Finding, raised bool, err error) {
rest, ok := strings.CutPrefix(c.Key, assignment.Module()+".")
if !ok || rest == "" {
return f, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
}
kind, id, _ := strings.Cut(rest, ".")
if id == "" {
id = c.Subject.ID
}
scope := c.Subject.Scope
if scope == "" {
scope = mcc.ScopeModule
}
if c.State == occ.StateCleared {
return mcseat.Finding{Scope: scope, ID: KeyID(id), Kind: kind}, false, nil
}
if c.State != occ.StateRaised {
return f, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
}
summary := c.Summary
if summary == "" {
summary = c.Headline
}
var actions []mcseat.Action
for _, a := range c.Actions {
actions = append(actions, mcseat.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
}
return Observed{Scope: scope, ID: id, Kind: kind, Machine: c.Subject.Machine, Severity: string(c.Severity), Summary: summary,
Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs, Actions: actions, Source: "assigner." + kind}.Finding(at), true, nil
}
// Findings is where the assigner's findings go: its finding writer's write and clear (design 47 G §689, A-29), by the
// finding's key `<scope>.<id>.<kind>`.
type Findings struct {
Put func(f mcseat.Finding) error
Delete func(key string) error
Keys func() ([]string, error)
Now func() time.Time
}
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
// findings bucket at all. A finding that cannot be written is the error.
func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
return func(c oc.Condition) error {
var errs []error
if f.Put != nil {
now := time.Now
if f.Now != nil {
now = f.Now
}
finding, raised, err := FindingOf(c, now())
key := views.FindingKey(finding.Scope, finding.ID, finding.Kind)
switch {
case err != nil:
errs = append(errs, err)
case raised:
if err := f.Put(finding); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
}
default:
if err := f.Delete(key); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err))
}
}
}
if f.Put == nil {
if say != nil {
if err := say(c); err != nil {
errs = append(errs, err)
}
}
}
return errors.Join(errs...)
}
}
// KindActingCycle is the finding of a want held on a cycle of acting waits no rule breaks (design 47 G A-323, A-330): scope module,
// id `<node>.<module>` of the held want, naming the cycle with each edge's step; cleared by the next pass that finds it held on no
// cycle.
const KindActingCycle = "acting-cycle"
// ActingCycleFinding is the finding acting-cycle of module's want on node, held on cycle.
func ActingCycleFinding(node, module, cycle string) Observed {
return Observed{Scope: mcc.ScopeModule, ID: node + "." + module, Kind: KindActingCycle, Machine: node, Severity: "warning",
Summary: fmt.Sprintf("the want of %s on %s is held on a cycle of acting waits: %s", module, node, cycle),
Headline: fmt.Sprintf("%s on %s waits on a cycle of acting parts", module, node),
Explanation: fmt.Sprintf("Each acting part on the cycle %s waits for another's new version to run, so none moves by itself. "+
"assigner_release lets one move with the version running.", cycle),
Source: "assigner." + KindActingCycle}
}
@@ -1,6 +0,0 @@
package assign
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
@@ -1,134 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
// PoliciesBucket is the assigner's state `policies`.
func PoliciesBucket() string { return assignment.Module() + "_" + buckets.PoliciesState }
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
out := map[string]catalogue.PersonPolicy{}
var bad []string
for key, raw := range values {
var p catalogue.PersonPolicy
if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key {
bad = append(bad, key)
continue
}
out[key] = p
}
sort.Strings(bad)
return out, bad
}
// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one.
func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) {
var c *catalogue.PersonPolicy
if p, ok := chosen[m.Module]; ok {
c = &p
}
return catalogue.Upgrade(m, c)
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
const KindProvisionFailed = "provision-failed"
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
func ProvisionFailedKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionFailed)
}
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Source: "assigner." + KindProvisionFailed}, at)
}
// encode is the observation as the assigner's finding at at.
func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil }
func also(provider, consumer string) []string {
if provider == "" || provider == consumer {
return nil
}
return []string{provider}
}
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
const KindProvisionMissing = "provision-missing"
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
func ProvisionMissingKey(key string) string {
return mcc.Key(mcc.ScopeProvider, key, KindProvisionMissing)
}
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) {
parts := strings.SplitN(key, ".", 3)
if len(parts) != 3 {
return mcseat.Finding{}, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
}
node, module, provision := parts[0], parts[1], parts[2]
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
"no provider of it may be placed", module, node, provision, bound, module)
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Source: "assigner." + KindProvisionMissing}, at)
}
// KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6).
const KindUnplaced = "unplaced"
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
func UnplacedKey(node string) string {
return mcc.Key(mcc.ScopeMachine, node, KindUnplaced)
}
// UnplacedFinding is the finding of a node no assignments names since the lease write.
func UnplacedFinding(node string, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s runs, but no assignments names it: since the controller handed its lease away nothing composes it, "+
"so the assigner writes it no want", node)
return encode(Observed{Scope: mcc.ScopeMachine, ID: node, Kind: KindUnplaced, Machine: node, Summary: summary,
Headline: clipHeadline(node + " has no assignments"), Needs: "place it with mesh-module-assigner.propose.",
Explanation: occ.NeedsYou + "place it with mesh-module-assigner.propose. " + summary + ".",
Source: "assigner." + KindUnplaced}, at)
}
func clipHeadline(s string) string {
if len([]rune(s)) <= 60 {
return s
}
return string([]rune(s)[:60])
}
@@ -2,16 +2,10 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"mesh-delivery-assigner/internal/puts"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
@@ -25,7 +19,7 @@ const ComposeWithin = 10 * time.Minute
// PutWorld is what one step of a put reads.
type PutWorld struct {
Node, Module string
Put puts.Put
Put delivery.Put
// Assigned says the module stands assigned to the node.
Assigned bool
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
@@ -36,7 +30,7 @@ type PutWorld struct {
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *ComposedDigest
Composed *delivery.ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
@@ -48,16 +42,13 @@ type PutWorld struct {
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *nodeengine.HaveByNodeModuleValue
// Running is the node-engine seat's last `running` of the module on the node; nil when it said none (design 47 G §309). A put
// ends on it, never on the have.
Running *nodeengine.Running
Now time.Time
Have *delivery.Have
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
type PutNext struct {
Put puts.Put
Put delivery.Put
// Ask writes the put's ask about the move (PutAsk).
Ask bool
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
@@ -69,7 +60,7 @@ type PutNext struct {
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
func StepPut(w PutWorld) PutNext {
p := w.Put
if puts.Ended(p.Step) {
if delivery.PutEnded(p.Step) {
return PutNext{Put: p}
}
say := func(step, format string, args ...any) PutNext {
@@ -77,21 +68,21 @@ func StepPut(w PutWorld) PutNext {
}
fp := short(p.FP)
if !w.Assigned {
return say(puts.Refused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its assignments first)", w.Module, w.Node)
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
}
switch w.Build {
case builds.BuildDone:
case builds.BuildFailed:
return say(puts.Refused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case delivery.BuildDone:
case delivery.BuildFailed:
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case "":
if p.FP != w.Newest {
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
}
return say(puts.Build, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
default:
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
}
if w.Composed == nil || !w.Granted {
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
@@ -99,33 +90,33 @@ func StepPut(w PutWorld) PutNext {
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
}
since := w.Now
if p.Step == puts.Compose && len(p.Steps) > 0 {
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
since = p.Steps[len(p.Steps)-1].At
}
if w.Now.Sub(since) > ComposeWithin {
return say(puts.Refused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
}
return say(puts.Compose, "waits up to %s: %s yet", ComposeWithin, missing)
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
}
d := *w.Composed
if p.Step == puts.Wanted || p.Step == puts.Holding {
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
return PutNext{Put: applied(w, p)}
}
p.Composed = d.Digest
if needsPerson(w, d) {
switch w.Answer {
case OptionNo:
return say(puts.Refused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
case OptionYes:
default:
if w.AskExpired {
return say(puts.Refused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
}
next := say(puts.Person, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
if p.Ask == "" {
next.Ask = true
next.Put.Ask, _ = asks.RecordAskKey(w.Node, w.Module, d.Digest)
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
}
return next
}
@@ -134,11 +125,11 @@ func StepPut(w PutWorld) PutNext {
case w.Snapshot != "":
p.Snapshot = w.Snapshot
case w.SnapshotFailed != "":
return say(puts.Refused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
case w.Snapshotting:
return say(puts.Snapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
default:
next := say(puts.Snapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next.Snapshot = true
return next
}
@@ -151,23 +142,23 @@ func StepPut(w PutWorld) PutNext {
if p.Snapshot != "" {
words += "; the streams were snapshotted: " + p.Snapshot
}
next := PutNext{Put: applied(w, p.Said(w.Now, puts.Wanted, words))}
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
next.Warranted = w.Bus && p.Ask != ""
return next
}
// applied is a wanted put once the node says how it went: holding or done when the node-engine seat's `running` names the put's
// fingerprint (design 47, *Compositions, wants and the apply*: from G a put ends on `running.<node>.<module>` naming its
// fingerprint, never on the have). A `running` without a fingerprint, or one that says the module stopped, ends none. The have
// still says a failed health check in the put's words.
func applied(w PutWorld, p puts.Put) puts.Put {
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP {
if p.Hold {
return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
}
return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
return p
}
if h := w.Have; h != nil && h.Fp == p.FP && h.Composed == p.Composed && failed(*h) {
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
return p
@@ -176,8 +167,8 @@ func applied(w PutWorld, p puts.Put) puts.Put {
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d ComposedDigest) bool {
runs := w.Have != nil && w.Have.Fp == w.Put.FP
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
@@ -187,22 +178,22 @@ func needsPerson(w PutWorld, d ComposedDigest) bool {
if !w.Recorded {
return false
}
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
func PutWant(p puts.Put) mwc.Want {
return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
func PutWant(p delivery.Put) delivery.Want {
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
func PutOwns(p puts.Put) (want *mwc.Want, owns bool) {
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
switch p.Step {
case puts.Wanted, puts.Holding:
case delivery.PutWanted, delivery.PutHolding:
w := PutWant(p)
return &w, true
case puts.Done, puts.Released, puts.Refused:
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
return nil, false
}
return nil, true
@@ -211,7 +202,7 @@ func PutOwns(p puts.Put) (want *mwc.Want, owns bool) {
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
func PutAsking(p puts.Put) int {
func PutAsking(p delivery.Put) int {
if n := p.At.UnixMilli(); n > 0 {
return int(n)
}
@@ -219,43 +210,43 @@ func PutAsking(p puts.Put) int {
}
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
func PutAct(node, module, fp, composed string) occ.Act {
return occ.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
func PutAct(node, module, fp, composed string) asks.Act {
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
}
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p puts.Put, d ComposedDigest, bus bool, now time.Time) (asks.Ask, error) {
yes := oc.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: occ.Acknowledge}
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + d.Words,
JudgedNeither: "recreates no container and moves no data",
JudgedUnchanged: "changes nothing the node runs",
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := occ.FormatTime(now.Add(7 * 24 * time.Hour))
expires := now.Add(7 * 24 * time.Hour)
if bus {
binds, err := occ.ActDigest(PutAct(node, module, p.FP, d.Digest))
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
if err != nil {
return asks.Ask{}, err
return delivery.AssignerAsk{}, err
}
yes.Level, yes.Binds = occ.Approve, binds
yes.Level, yes.Binds = asks.Approve, binds
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
expires = occ.FormatTime(now.Add(occ.ApproveLasts))
expires = now.Add(asks.ApproveLasts)
}
return assignerAsk(PutAsking(p), oc.Ask{
ID: meshdeliveryclient.AskingID(key, PutAsking(p)),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, PutAsking(p)),
Headline: fmt.Sprintf("Put %s on %s?", module, node),
Explanation: explanation,
Options: []oc.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: occ.Acknowledge}},
Who: occ.Operator,
Options: []asks.Option{yes,
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
Who: asks.Operator,
Expires: expires,
OnExpiry: "the node keeps what it runs",
About: key,
}), nil
}, Asking: PutAsking(p)}, nil
}
@@ -1,246 +0,0 @@
package assign
import (
"fmt"
"maps"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assignment"
plc "mesh-delivery-assigner/internal/assignment"
)
// The seat mesh-module-assigner (novox/hq ADR 0567), held by the assigner beside its wants: assignments is its state
// `assignments`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current assignments and the
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
// digest.
type ProposeIn struct {
ID, By, Why string
At time.Time
Changes []mma.ProposalChange
Current map[string]mma.Assignments
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
Warrant *mma.Warrant
// SeatListRead says the host has read the seat list from the seat mesh-catalog (ADR 0636 §1); until then a mesh seat
// the bootstrapped definitions do not name is not judged held once (plc.CheckAssignments).
SeatListRead bool
}
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
// the laid assignments of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
// CheckAssignments refuses of the result laid over every assignments (two modules on one host port, a provider not placed,
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Assignments, []string) {
p := mma.Proposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]int64{}, State: mmac.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
}
if p.Why == "" {
findings = append(findings, "a proposal says why: it is shown to the operator")
}
if len(in.Changes) == 0 {
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if c.Op == mmac.OpSeed {
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
}
}
laid, lay := plc.LayChanges(in.Current, in.Changes)
findings = append(findings, lay...)
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
all := map[string]mma.Assignments{}
for node, pl := range in.Current {
all[node] = pl
}
out := map[string]*mma.Assignments{}
for node, pl := range laid {
if before, ok := in.Current[node]; ok && mmac.SameAssignments(before, pl) {
continue
}
pl := pl
all[node] = pl
out[node] = &pl
p.Base[node] = int64(in.Revisions[node])
}
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
// candidate is no assignments, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
for _, node := range plc.RemovedNodes(p) {
if _, ok := in.Current[node]; !ok {
continue
}
delete(all, node)
out[node] = nil
p.Base[node] = int64(in.Revisions[node])
}
if len(findings) == 0 && len(out) == 0 {
findings = append(findings, "the changes leave every assignments as it is")
}
held, recorded := in.Held, in.Recorded
if held == nil {
held = func(string) bool { return false }
}
if recorded == nil {
recorded = func(string) bool { return false }
}
// A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
joining := map[string]bool{}
for _, c := range in.Changes {
if c.Op == mmac.OpAddNode {
joining[c.Node] = true
}
}
if len(joining) > 0 {
was := recorded
recorded = func(node string) bool { return joining[node] || was(node) }
}
for _, r := range plc.CheckAssignments(all, held, recorded, in.SeatListRead) {
findings = append(findings, r)
}
return p, out, findings
}
// ControllerModule is the controller's module, which nothing places again (issue 1126).
const ControllerModule = "mesh-controller"
// ControllerEnded is why assignments of the controller is refused: the parts that acted only once its lease was handed
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
// would write beside them, silently.
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
// whole assignments, or in what the changes lay. None when nothing does.
func ControllerRefusals(changes []mma.ProposalChange, laid map[string]mma.Assignments) []string {
var out []string
nodes := map[string]bool{}
for i, c := range changes {
places := c.Module == ControllerModule && c.Op != mmac.OpTakeOff
if c.Whole != nil && slices.Contains(mmac.Assigned(*c.Whole), ControllerModule) {
places = true
}
if places {
out = append(out, fmt.Sprintf("change %d places %s on %s: %s", i+1, ControllerModule, c.Node, ControllerEnded))
nodes[c.Node] = true
}
}
for _, node := range slices.Sorted(maps.Keys(laid)) {
if !nodes[node] && slices.Contains(mmac.Assigned(laid[node]), ControllerModule) {
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
}
}
return out
}
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
// mesh-sdk mmac.Keys.Writes), the revision of the node's assignments it is written at (0: it has none, so
// it is created) and its value (nil: it is deleted).
type Write struct {
Node, Key string
Base uint64
Value *mma.Assignments
}
// Writes is every key a proposal writes, by node order.
func Writes(p mma.Proposal, laid map[string]*mma.Assignments) ([]Write, error) {
var out []Write
for _, node := range plc.Nodes(p) {
key, err := mmac.NodeKey(node)
if err != nil {
return nil, err
}
out = append(out, Write{Node: node, Key: key, Base: uint64(p.Base[node]), Value: laid[node]})
}
return out, nil
}
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
// the controller's verbs (ADR 0567 §8, issue 774).
const (
KindControllerCall = "controller-call"
KindControllerPin = "controller-pin"
)
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
// module, or node and provision for a pin).
func CallClearedCondition(kind, node, module string) oc.Condition {
return oc.Condition{Key: assignment.Module() + "." + kind + "." + node + "." + module,
Subject: oc.ConditionSubject{Scope: "module", ID: module, Machine: node}, State: occ.StateCleared,
Resolved: "the assigner calls no verb of the controller any more: assignments is the one source."}
}
// AssignmentsAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
func AssignmentsAct(p mma.Proposal) occ.Act {
return occ.Act{"assignments": "write", "proposal": p.ID, "changes": plc.ChangesDigest(p)}
}
// AssignmentsAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
// another asking, under an id of its own.
func AssignmentsAsking(p mma.Proposal, at time.Time) int {
return int(at.Unix())
}
// AssignmentsAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
func AssignmentsAsk(key string, p mma.Proposal, brief string, asking int, now time.Time) (asks.Ask, error) {
binds, err := occ.ActDigest(AssignmentsAct(p))
if err != nil {
return asks.Ask{}, err
}
nodes := plc.Nodes(p)
headline := fmt.Sprintf("Change assignments on %s?", strings.Join(nodes, ", "))
if n := len([]rune(headline)); n > occ.HeadlineLength {
headline = fmt.Sprintf("Change assignments on %d machines?", len(nodes))
}
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: headline,
Explanation: brief,
Options: []oc.Option{
{ID: OptionYes, Label: "Approve", Does: "the assignments of " + strings.Join(nodes, ", ") + " is written as proposed",
Level: occ.Approve, Binds: binds},
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(occ.ApproveLasts)),
OnExpiry: "nothing is written, and the proposal ends",
About: key,
}), nil
}
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
func Rest(p mma.Proposal, written map[string]*mma.Assignments) (mma.Proposal, bool) {
rest := p
rest.Changes, rest.Base = nil, map[string]int64{}
for _, c := range p.Changes {
if _, done := written[c.Node]; !done {
rest.Changes = append(rest.Changes, c)
}
}
for node, base := range p.Base {
if _, done := written[node]; !done {
rest.Base[node] = base
}
}
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", mmac.ProposalAnswering
return rest, len(rest.Changes) > 0
}
@@ -1,107 +0,0 @@
package assign
import (
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/keys"
"slices"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
plc "mesh-delivery-assigner/internal/assignment"
)
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestTheAssignmentsSeatDecides(t *testing.T) {
anchor := mma.Assignments{V: keys.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []mma.PlacedModule{{Module: "store", Ports: map[string]int64{"5432": 5432}}, {Module: "web"}}}
laptop := mma.Assignments{V: keys.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
in := func(changes ...mma.ProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]mma.Assignments{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Held: func(m string) bool { return m != "ghost" },
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
}
move := []mma.ProposalChange{{Op: mmac.OpTakeOff, Node: "anchor", Module: "web"}, {Op: mmac.OpPlace, Node: "laptop", Module: "web"}}
p, laid, findings := Propose(in(move...))
if len(findings) != 0 || p.State != mmac.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(mmac.Assigned(*laid["laptop"]), []string{"shell", "web"}) || !slices.Equal(mmac.Assigned(*laid["anchor"]), []string{"store"}) {
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
}
if _, err := plc.EncodeProposal(p); err != nil {
t.Fatalf("the proposal cannot be written: %v", err)
}
for name, c := range map[string]struct {
in ProposeIn
want string
}{
"a module not held": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "anchor", Module: "web"}), "leave every assignments as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
// act without its lease.
"the controller placed again": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "anchor", Whole: &mma.Assignments{V: keys.Version, Node: "anchor",
Decided: anchor.Decided, Modules: []mma.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
"a pin of nobody placed": {in(mma.ProposalChange{Op: mmac.OpPin, Node: "laptop", Provision: "db", Provider: &mma.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
}
}
// Every kind of change is taken: the assignments is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
for _, c := range []mma.ProposalChange{
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: mmac.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int64{"80": 8080}, Bindings: map[string]mma.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: mmac.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: mmac.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
if _, _, f := Propose(in(c)); len(f) != 0 {
t.Errorf("%s is refused: %v", c.Op, f)
}
}
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
if rest, ok := Rest(p, map[string]*mma.Assignments{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != mmac.ProposalAnswering || rest.Must != nil {
t.Errorf("the rest = %+v, %v", rest, ok)
}
if _, ok := Rest(p, laid); ok {
t.Error("a proposal written whole leaves something to write")
}
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
for _, unbound := range [][]string{{plc.PulsesBucket()}, nil} {
c := SeatWaitsCondition(unbound)
if _, err := occ.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == occ.StateRaised) ||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+plc.PulsesBucket())) {
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
}
}
writes, err := Writes(p, laid)
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
t.Fatalf("writes = %+v, %v", writes, err)
}
a, err := AssignmentsAsk("assignments.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || occ.Check(mustAsk(t, a), time.Unix(100, 0)) != nil || mustAsk(t, a).Options[0].Level != occ.Approve || occ.Performs(mustAsk(t, a).Options[0], AssignmentsAct(p)) != nil {
t.Fatalf("the ask = %+v, %v", a, err)
}
q := p
q.Changes = append(slices.Clone(p.Changes), mma.ProposalChange{Op: mmac.OpUnpin, Node: "laptop", Provision: "x"})
if occ.Performs(mustAsk(t, a).Options[0], AssignmentsAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
}
@@ -1,38 +0,0 @@
package assign
import (
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
)
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
func (m Module) verbWaiting(node string) (string, bool) {
if m.Served == nil {
return "", false
}
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
if !m.Served(v) {
return graph.VerbWaitWords(v, m.Name, node), true
}
}
return "", false
}
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
func (m Module) busStateWaiting(node string) (string, bool) {
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
return "", false
}
return graph.BusStateWaitWords(m.Name, node), true
}
@@ -2,39 +2,24 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"maps"
"mesh-delivery-assigner/internal/puts"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
Name string
FP string
Built bool
// BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole
// (design 47 G §575, A-162): the words of a hold under rule 1.
BuildFailed bool
BuildError string
Held bool
Granted bool
Name string
FP string
Built bool
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
@@ -43,33 +28,10 @@ type Module struct {
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
GrantedOn func(node string) bool
Manifest catalogue.Manifest
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
Provisions func(node string) map[string]meshprovisionerclient.Provision
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
Answering func(provision string) []catalogue.Manifest
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
// is waited on for want of an offer.
Offered func(provision string) bool
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
// served (Served). Nil: no graph node is read, and no want waits on a verb.
Invokes []string
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
Served func(invoke string) bool
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
// runs, and no want waits on bus state.
Offering func(provision string) bool
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
Chosen *catalogue.PersonPolicy
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (ComposedDigest, bool)
Nodes []string
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
@@ -83,88 +45,12 @@ type Module struct {
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]puts.Put
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
Needs []string
// FormNeeds are the reference forms the module's build at FP is written in (`form:<name>`, the builder's build key's
// needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the
// build names none, and holds nothing.
FormNeeds []string
// Self says whether node composes the module itself: the catalogue's assignments names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the assignments (issue 1126). Nil: no node does.
Self func(node string) bool
// KeptBuilds answers what a want at a fingerprint names of its build: its parts and its keeping node as the builder's `build.ended`
// names them, or the released version and parts of the catalog's `version-released` (design 47 G §379, §526, A-96), for a node that states the capability WantParts. Nil: none is read.
KeptBuilds func(fp string) (KeptBuild, bool)
// MachineHeld says whether the node's own network condition is open, which holds the machine as a whole: no want moves to it
// until that clears (design 47 G §706 *M's own network*). Nil: none is held.
MachineHeld func(node string) bool
// Acting are the acting edges of the assignment graph entry of this module on node that the assigner holds (composing, placing)
// and whose wait was not released by hand (design 47 G §266, §268): the want moves only once each acting part's version runs
// on the edge's node, as Nodes.Running says it. Nil: none.
Acting func(node string) []ActingEdge
Puts map[string]delivery.Put
}
// ActingEdge is one acting edge: Part's version Fp acts in Step on Node, before the entry moves.
type ActingEdge struct{ Node, Part, Fp, Step string }
// Runs says the part's version of the edge runs on its node, as the node-engine seat's `running` says it.
func (n Nodes) Runs(e ActingEdge) bool {
if n.Running != nil {
r, ok := n.Running[e.Node][e.Part]
return ok && !r.Stopped && r.Fingerprint == e.Fp
}
h, ok := n.Haves[e.Node][e.Part]
return ok && healthy(h, e.Fp)
}
// actingWaiting says why the module's want on node waits for an acting part's new version (§266): the first edge not run.
func (m Module) actingWaiting(node string, n Nodes) (string, bool) {
if m.Acting == nil {
return "", false
}
for _, e := range m.Acting(node) {
if !n.Runs(e) {
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", m.Name, node, e.Step, e.Part, short(e.Fp), e.Node), true
}
}
return "", false
}
// WantParts is the capability a node-engine of G states (ADR 0614 §4, design 47 G §525): it reads a want's `parts` and `kept-on`.
const WantParts = "want-parts"
// KeptBuild is what a want names of a build: the digest of each of its parts by `<module>:<artifact>`, and, as the builder's
// `build.ended` says it, the build node that keeps a kept build; for a trunk build the released version the catalog's
// `version-released` says (design 47 G §379, §526, A-96).
type KeptBuild struct {
Parts map[string]string
KeptOn string
Version string
}
// withKept is w naming the kept build at its fingerprint, the parts by digest and the keeping node, only while the node's
// engine states WantParts (design 47 G §525, §526); a want of a build that is not kept, or of a node that does not state it,
// is w.
func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want {
if m.KeptBuilds == nil {
return w
}
k, ok := m.KeptBuilds(w.Fp)
if !ok || !slices.Contains(n.Reads[node], WantParts) {
return w
}
w.Parts, w.KeptOn, w.Version = k.Parts, k.KeptOn, k.Version
return w
}
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
return m.Composed(node, fp)
}
@@ -172,26 +58,16 @@ func (m Module) composition(node, fp string) (ComposedDigest, bool) {
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Running is, by node and module, the last `running` the node-engine seat said (design 47 G §309): a provider runs healthy
// while the build it runs does, as this says it, whatever a newer want of it waits for (rule 5, main file *A provider
// before its consumers*). Nil: the seat's events are not heard, and the have says it.
Running map[string]map[string]nodeengine.Running
Haves map[string]map[string]delivery.Have
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
func healthy(h delivery.Have, fp string) bool {
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}
// Failed says whether a have's health check failed once the node is done moving.
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
// name, ADR 0528 clause 3). Empty when none is online.
func Canary(nodes []string, online map[string]bool) string {
@@ -206,170 +82,48 @@ func Canary(nodes []string, online map[string]bool) string {
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
// down and one module never blocks another.
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
// without which there is nothing to want: the build done, not held, the grant and the composition.
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
if m.BuildFailed {
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError)
}
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP))
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
case m.Held:
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
case m.MachineHeld != nil && m.MachineHeld(node):
return false, mwc.RuleNotHeld, fmt.Sprintf("the condition machine.%s.network is open: no want moves to %s until its own network is healthy", node, node)
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
}
if !m.Granted {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if words, lacking := m.readerLacking(node, n); lacking {
return false, mwc.RuleComposed, words
}
if _, ok := m.composition(node, m.FP); !ok {
return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.actingWaiting(node, n); waiting {
return false, mwc.RuleComposed, words
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, mwc.RuleProvisions, words
}
if words, waiting := m.verbWaiting(node); waiting {
return false, mwc.RuleProvisions, words
}
if words, waiting := m.busStateWaiting(node); waiting {
return false, mwc.RuleProvisions, words
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
return true, 0, ""
}
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
// manifest needs that the engine does not state, read as the baseline when it states none, a bootstrapped seat counting as
// read by every engine (graph.MissingReads, ADR 0636 §2). It is held under rule 7 (the
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
//
// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a
// first form, so a build written in one is never held by its forms; the words name the forms it reads none of.
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
missing := graph.MissingReads(m.Needs, n.Reads[node])
if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) {
missing = slices.Clone(m.FormNeeds)
}
if len(missing) == 0 {
return "", false
}
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
}
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
func (m Module) provisionWaiting(node string) (string, bool) {
if m.Provisions == nil {
return "", false
}
if missing := m.MissingProvisions(node); len(missing) > 0 {
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
}
keyed := m.Provisions(node)
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
// bound (ADR 0603 §8).
needed := m.Manifest.Requires
if m.Answering != nil {
needed = catalogue.Awaited(m.Manifest, m.Answering)
have := map[string]bool{}
for _, p := range keyed {
have[p.Provision] = true
}
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
continue
}
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
continue
}
needed = append(needed, r)
if !have[r] {
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
}
}
}
for _, key := range slices.Sorted(maps.Keys(keyed)) {
if p := keyed[key]; p.State != meshprovisionerclient.Provided && slices.Contains(needed, p.Provision) {
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
if p.Why != "" {
words += ": " + p.Why
}
return words, true
}
}
return "", false
}
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
func (m Module) MissingProvisions(node string) []string {
if m.Provisions == nil || m.Answering == nil {
return nil
}
have := map[string]bool{}
for _, p := range m.Provisions(node) {
have[p.Provision] = true
}
var out []string
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
if !have[r] {
out = append(out, r)
}
}
return out
}
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
func (n Nodes) runsHealthy(node, module string) bool {
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy)
}
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
// module of which nothing was heard counts as not healthy. The have is never read for it.
func (n Nodes) RunsHealthy(node, module, fp string) bool {
r, ok := n.Running[node][module]
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
@@ -389,24 +143,13 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if at != "" && n.runsHealthy(at, p) {
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
return true, ""
}
}
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
func unreached(m Module, node string, n Nodes) string {
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return why
}
}
return ""
}
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
@@ -420,15 +163,12 @@ func keepsData(m catalogue.Manifest) bool {
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
type Outcome struct {
Wants map[string]mwc.Want
Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *mwc.Held
Held *delivery.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
NodeFailed map[string]NodeFailure
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
@@ -436,9 +176,6 @@ type Outcome struct {
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
Unreached map[string]string
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
@@ -451,54 +188,34 @@ type Outcome struct {
type RecordNeed struct {
Node string
Digest string
Judgement Judgement
Judgement delivery.Judgement
Words string
}
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
// §2): no bar of its own.
// Holding is a rule holding a node back, and why.
type Holding struct {
Rule int
Words string
}
// Reason is the holding in words: the rule and its words.
func (h Holding) Reason() string {
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.Fp != m.FP || !failed(h) {
continue
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
// §2, ADR 0429 §6).
if out.NodeFailed == nil {
out.NodeFailed = map[string]NodeFailure{}
}
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
}
mm := m
@@ -508,24 +225,24 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
break
}
before, had := wants[node]
var next mwc.Want
var next delivery.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: puts.Holding}, true
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
}
if put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp {
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
w.Composed = ul.Composed
}
if !had || !mwc.SameWant(normal(before), normal(w)) {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
}
@@ -536,75 +253,60 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !mwc.SameWant(normal(before), normal(w)) {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
if older {
_, composed := m.composition(node, have.Fp)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
}
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
switch {
case older:
ok, target = true, have.Fp
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
if why := unreached(m, node, n); why != "" {
if out.Unreached == nil {
out.Unreached = map[string]string{}
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) &&
n.Haves[node][m.Name].Fp != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
next = n.withKept(m, node, mwc.Want{Fp: target, Composed: d.Digest})
next = delivery.Want{FP: target, Composed: d.Digest}
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, int64(rule), words
next.Newest, next.Rule, next.Words = m.FP, rule, words
at.FP = target
}
// A placed node's want names the build alone: it is the same target while the build is.
if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = mwc.Want{}, false
asked, hadLive = delivery.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
case held && had && before.Fp == m.FP:
case held && had && before.FP == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
@@ -622,7 +324,7 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words}
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
@@ -633,31 +335,31 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
case there && h.Phase != delivery.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
case had && (!there || h.Phase == delivery.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)}
if !mwc.SameWant(normal(before), normal(next)) {
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words}
if before.Fp == m.FP {
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = mwc.Want{Fp: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed {
next = delivery.Want{FP: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.Fp
at.FP = before.FP
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -666,59 +368,18 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
}
}
}
next, write := m.selfDerived(node, before, had, next, n, &out)
if !write {
continue
}
if !had || !mwc.SameWant(normal(before), normal(next)) {
if !had || !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
for node, before := range wants {
if !slices.Contains(m.Nodes, node) && !before.Removed {
out.Wants[node] = mwc.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
}
}
return out
}
// selfDerived is next with its Self derived from the assignments now (Module.Self), whichever rule made it, and whether it is
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
// assignments seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) {
switch {
case next.Removed:
return next, true
case m.selfOn(node):
next.Self, next.Composed = true, ""
return next, true
case !next.Self && next.Composed != "":
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.Fp)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))}
return next, false
}
at := m
at.FP = next.Fp
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
return next, false
}
next.Composed = d.Digest
return next, true
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
@@ -729,50 +390,13 @@ func NoWantWords(rule int, words string) string {
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h nodeengine.HaveByNodeModuleValue) bool {
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
func healthyElsewhere(m Module, n Nodes, node string) []string {
var out []string
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
out = append(out, other)
}
}
return out
}
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
return true
}
}
return false
}
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
func FailedAloneWords(module, fp string, failing []string) string {
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
strings.Join(failing, ", "))
}
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
type NodeFailure struct {
Others []string
Words string
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h nodeengine.HaveByNodeModuleValue) bool {
return h.Phase == nodeengineclient.PhaseRunning
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
@@ -784,10 +408,10 @@ func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
continue
}
if !n.RunsHealthy(node, m.Name, m.FP) {
if !healthy(h, m.FP) {
return false
}
ran++
@@ -797,7 +421,7 @@ func allRunHealthy(m Module, n Nodes) bool {
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
func HeldWords(h delivery.Have, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
@@ -814,35 +438,43 @@ func LiftedWords(module, fp string) string {
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w mwc.Want) mwc.Want {
func normal(w delivery.Want) delivery.Want {
w.V = 0
return w
}
func short(fp string) string {
part, err := catalogkey.KeyPart(fp)
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
return part[:12]
}
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) {
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
@@ -851,16 +483,11 @@ func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running str
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
if d.Policy != "" {
return d.Policy == catalogue.PolicyRecord
return d.Policy == delivery.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
@@ -869,27 +496,27 @@ func (m Module) recorded(d ComposedDigest) bool {
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) {
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return mwc.Want{}, false
return delivery.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) {
return mwc.Want{}, false
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
return delivery.Want{}, false
}
d, ok := m.composition(node, running.Fp)
d, ok := m.composition(node, running.FP)
if !ok || !d.UserListOnly {
return mwc.Want{}, false
return delivery.Want{}, false
}
w := mwc.Want{Fp: running.Fp, Composed: d.Digest}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
return w, true
}
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g grantsstate.Grant, node, module string) bool {
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
return g.NamesNode(node, module)
}
File diff suppressed because it is too large Load Diff
@@ -1,138 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// A candidate's assignment graph (design 47 G §189, §190, A-201, A-235): computed per run on request from the candidate's catalog
// graph revision, kept apart from trunk's under the seat's key form candidate.<group>.<heads>, the group `run-<id>`, and dropped on
// the runner's group-ended of that run; it writes no want, and the gate's heads-replaced drops nothing.
// CandidateObject is the object name of a candidate's revision (the seat's key form candidate.<group>.<heads>).
func CandidateObject(group, heads string) string {
return meshwants.AssignmentRevisionsByCandidateGroupHeadsKey(group, heads)
}
// KeepCandidate keeps c as the candidate's revision of group and heads, numbered after the one kept of it, unless that one lists
// the same entries from the same inputs; its entry objects are written first, an entry trunk shares stored once.
func KeepCandidate(ctx context.Context, o Objects, c Computed, group, heads, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
if group == "" || heads == "" {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("a candidate's revision names its group and its heads, not %q and %q", group, heads)
}
was, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
has := err == nil
if err != nil && !errors.Is(err, ErrNoObject) {
return was, false, err
}
if has && Same(was, c.Revision) {
return was, false, nil
}
rev := c.Revision
rev.Kind, rev.Group, rev.Heads, rev.Cause = meshcatalogclient.KindCandidate, group, heads, cause
rev.Revision = was.Revision + 1
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return was, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return was, false, err
}
if err := o.Put(ctx, CandidateObject(group, heads), list); err != nil {
return was, false, fmt.Errorf("the candidate %s's revision in %s: %w", group, RevisionsObjects, err)
}
return rev, true, nil
}
// ReadCandidate is the candidate's revision kept of group, of heads when named, else the newest kept of the group; ErrNoCandidate
// when none is kept.
func ReadCandidate(ctx context.Context, o Objects, group, heads string) (meshwants.AssignmentRevision, error) {
if heads != "" {
rev, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
if errors.Is(err, ErrNoObject) {
err = fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads)
}
return rev, err
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return meshwants.AssignmentRevision{}, err
}
var newest meshwants.AssignmentRevision
found := false
for _, n := range names {
rev, err := readCandidateObject(ctx, o, n)
if err != nil {
return rev, err
}
if !found || rev.At != nil && newest.At != nil && rev.At.After(*newest.At) {
newest, found = rev, true
}
}
if !found {
return newest, fmt.Errorf("%w (group %q)", ErrNoCandidate, group)
}
return newest, nil
}
// CandidateNames is the object name of every revision kept of the candidate group; every candidate's when group is "".
func CandidateNames(ctx context.Context, o Objects, group string) ([]string, error) {
names, err := o.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var out []string
for _, n := range names {
g, _, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if ok && (group == "" || g == group) {
out = append(out, n)
}
}
return out, nil
}
// Remover is an object bucket that also removes an object.
type Remover interface {
Objects
// Delete removes the object of that name; one not held is no error.
Delete(ctx context.Context, name string) error
}
// DropCandidate removes every revision kept of the candidate group (its run's group-ended, A-235); the entries it alone listed are
// removed with the next revisions nothing pins. It says how many it removed.
func DropCandidate(ctx context.Context, o Remover, group string) (int, error) {
if strings.TrimSpace(group) == "" {
return 0, errors.New("a candidate to drop names its group")
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return 0, err
}
for i, n := range names {
if err := o.Delete(ctx, n); err != nil {
return i, fmt.Errorf("the candidate %s's revision %s: %w", group, n, err)
}
}
return len(names), nil
}
func readCandidateObject(ctx context.Context, o Objects, name string) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, name)
if err != nil {
return rev, err
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("the candidate's revision %s: %w", name, err)
}
return rev, nil
}
@@ -1,303 +0,0 @@
// Package assigngraph is the assigner's assignment graph (design 47, *Two graphs, one source of truth*; ADR 0656): the
// module versions placed on nodes, with the edges between versions, computed from the catalog graph of the branch each
// placement names, the placements and the nodes' capability readings, and kept per revision as objects of the seat
// mesh-wants' kept data assignment-revisions.
package assigngraph
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/assignment"
)
// The kinds of an edge of the assignment graph (ADR 0656; the seat's Edge).
const (
EdgeProvisioning = "provisioning"
EdgeContribution = "contribution"
EdgeActing = "acting"
)
// Inputs are what a revision is computed from, and nothing else (design 47 lines 1353-1356).
type Inputs struct {
// Halves is the catalog graph revision held of each watched branch, by branch.
Halves map[string]meshcatalogclient.Half
// Placements is every node's placement key, by node.
Placements map[string]mma.Assignments
// FirstBranch is the first watched branch of a module's registration, the branch a placement naming none runs
// (design 47 lines 522-523); "" when no registration is known for it.
FirstBranch func(module string) string
// Capabilities is each node's capability reading, by its digest.
Capabilities []meshwants.AssignmentRevisionCapabilitiesItem
// Definitions are the seat definitions read whole through the seat mesh-catalog's verb seat-definitions, by seat: of
// them, those an entry's catalog graph revision names in its definitions say which seat delivers a requirement (§708).
Definitions map[string]seatdef.Definition
}
// Computed is one assignment graph revision as computed, before it is numbered and kept.
type Computed struct {
Revision meshwants.AssignmentRevision
// Entries is every entry the revision lists, by its digest.
Entries map[string]meshwants.AssignmentEntry
// Uncomputed is each placed module whose entry could not be computed, as "<node>/<module>: why", sorted: it is said,
// never left out in silence.
Uncomputed []string
// Cycles are the cycles among the acting waits, within one step or across steps (Cycles, design 47 G §272, A-323): said, never
// left in silence.
Cycles []string
}
// placedAt is one placed module with what its entry is computed from.
type placedAt struct {
node string
placed mma.PlacedModule
branch string
entry meshwants.AssignmentEntry
langs map[string]bool
acts []actingRule
// provides is what its catalog entry provides; contributes the requirements it carries contributed values to (§709).
provides, contributes []string
// delivering is, by requirement, the seat that delivers it among the definitions its revision names (§708).
delivering map[string]seatdef.Definition
// optional says a requirement is one it runs without (ADR 0330 §3), by the entry's manifest (A-229).
optional func(string) bool
}
type actingRule struct{ seat, step, on, bound string }
// Compute is the assignment graph revision of in: one entry per module placed on a node whose branch's catalog graph
// holds it, its version the build the placement names or else the catalog entry's fingerprint (design 47 lines 909-914),
// with the edges of provisioning to the providers its placement binds or its node pins, else the provider on its node or
// the only provider (an entry held naming the candidates where none is), and of acting from each acting
// part's version to each version its rule names (design 47 *An acting part's new version acts first*: "the assigner
// gives an edge from the acting part's version to each version its rule names").
func Compute(in Inputs) Computed {
out := Computed{Entries: map[string]meshwants.AssignmentEntry{}}
var placed []*placedAt
branches := map[string]bool{}
for _, node := range slices.Sorted(maps.Keys(in.Placements)) {
mods := slices.Clone(in.Placements[node].Modules)
slices.SortFunc(mods, func(a, b mma.PlacedModule) int { return strings.Compare(a.Module, b.Module) })
for _, m := range mods {
branch := m.Branch
if branch == "" && in.FirstBranch != nil {
branch = in.FirstBranch(m.Module)
}
if branch == "" {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: no watched branch: its placement names none and no registration of it is known", node, m.Module))
continue
}
half, ok := in.Halves[branch]
if !ok {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph of the branch %s is not held", node, m.Module, branch))
continue
}
ce, ok := half.Entries[m.Module]
if !ok {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph revision %d of %s holds no entry of it", node, m.Module, half.Revision.Revision, branch))
continue
}
digest := ""
for _, ref := range half.Revision.Entries {
if ref.Module == m.Module {
digest = ref.Digest
}
}
if digest == "" {
out.Uncomputed = append(out.Uncomputed, fmt.Sprintf("%s/%s: the catalog graph revision %d of %s lists no digest of its entry", node, m.Module, half.Revision.Revision, branch))
continue
}
fp := ce.Fp
if m.Build != "" {
fp = m.Build
}
p := &placedAt{node: node, placed: m, branch: branch, langs: map[string]bool{},
entry: meshwants.AssignmentEntry{V: keys.Version, Node: node, Module: m.Module, Branch: branch, Fp: fp, Catalog: digest}}
for lang := range ce.Toolchains {
p.langs[lang] = true
}
for _, r := range ce.Acts {
p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On, bound: r.Bound})
}
p.provides, p.contributes = ce.Provides, ce.Contributes
p.delivering = map[string]seatdef.Definition{}
for _, d := range half.Revision.Definitions {
if def, ok := in.Definitions[d.Seat]; ok && def.Delivers != "" {
if _, had := p.delivering[def.Delivers]; !had {
p.delivering[def.Delivers] = def
}
}
}
p.optional = func(string) bool { return false }
if man, err := manifestOf(ce); err == nil {
p.optional = man.IsOptional
}
placed = append(placed, p)
branches[branch] = true
}
}
// Each requirement points, by design 27's order (design 47 G §357, §708), to the named seat's holder, else a pin, else
// the provider on the consumer's node, else the only provider; otherwise the entry is held, naming the candidates, and
// nothing is guessed. A requirement contributes names gets a contribution edge to the same provider (§709).
holders := assignment.SeatHoldersOf(in.Placements)
for _, p := range placed {
ce := in.Halves[p.branch].Entries[p.placed.Module]
for _, r := range ce.Requires {
prov, held := resolve(p, r, placed, holders, in.Placements[p.node].Pins)
switch {
case held != nil:
p.entry.Held = append(p.entry.Held, *held)
case prov.Module != "":
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: prov.Node, Module: prov.Module, Provision: r})
if slices.Contains(p.contributes, r) {
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeContribution, Node: prov.Node, Module: prov.Module, Provision: r})
}
}
}
slices.SortFunc(p.entry.Held, func(a, b meshwants.Held) int { return strings.Compare(a.Requirement, b.Requirement) })
}
at := map[string]*placedAt{}
for _, p := range placed {
at[p.node+"/"+p.placed.Module] = p
}
for _, p := range placed {
for i, e := range p.entry.Edges {
if prov, ok := at[e.Node+"/"+e.Module]; ok {
p.entry.Edges[i].Fp = prov.entry.Fp
}
}
}
// Acting: an acting part on the entry's own node acts there (a node's seat is held on every node); else every node it
// is placed on (a seat held once for the mesh is held on one).
byModule := map[string][]*placedAt{}
for _, p := range placed {
byModule[p.placed.Module] = append(byModule[p.placed.Module], p)
}
for _, part := range slices.Sorted(maps.Keys(byModule)) {
holders := byModule[part]
for _, rule := range holders[0].acts {
for _, t := range placed {
if t.placed.Module == part || (rule.on != "" && rule.on != "every" && !t.langs[rule.on]) {
continue
}
from := holders
for _, h := range holders {
if h.node == t.node {
from = []*placedAt{h}
}
}
for _, h := range from {
t.entry.Edges = append(t.entry.Edges, meshwants.Edge{Kind: EdgeActing, Node: h.node, Module: part, Fp: h.entry.Fp,
Seat: rule.seat, Step: rule.step, Bound: rule.bound})
}
}
}
}
rev := meshwants.AssignmentRevision{V: keys.Version, Kind: meshcatalogclient.KindTrunk, Placements: assignment.ExportDigest(in.Placements),
Capabilities: in.Capabilities, Entries: []meshwants.NodeEntryRef{}}
for _, b := range slices.Sorted(maps.Keys(branches)) {
h := in.Halves[b]
rev.Catalogs = append(rev.Catalogs, meshwants.CatalogOf{Branch: b, Revision: h.Revision.Revision, Source: h.Revision.Source})
}
for _, p := range placed {
sortEdges(p.entry.Edges)
d := EntryDigest(p.entry)
out.Entries[d] = p.entry
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: p.node, Module: p.placed.Module, Digest: d})
}
out.Revision = rev
out.Cycles = Cycles(slices.Collect(maps.Values(out.Entries)))
slices.Sort(out.Uncomputed)
return out
}
func sortEdges(edges []meshwants.Edge) {
slices.SortFunc(edges, func(a, b meshwants.Edge) int {
return strings.Compare(edgeKey(a), edgeKey(b))
})
}
func edgeKey(e meshwants.Edge) string {
return strings.Join([]string{e.Kind, e.Node, e.Module, e.Provision, e.Seat, e.Step, e.Fp, e.Bound}, "\x00")
}
// EntryDigest is the digest an entry is stored under: the same entry, the same digest.
func EntryDigest(e meshwants.AssignmentEntry) string {
raw, _ := json.Marshal(e)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Same says two revisions list the same entries computed from the same inputs, so the second is no new revision.
func Same(a, b meshwants.AssignmentRevision) bool {
return slices.Equal(a.Entries, b.Entries) && slices.Equal(a.Catalogs, b.Catalogs) && a.Placements == b.Placements &&
slices.EqualFunc(a.Capabilities, b.Capabilities, func(x, y meshwants.AssignmentRevisionCapabilitiesItem) bool {
// A reading's digest is over all of it: its readings and its runner's fingerprint among it.
return x.Node == y.Node && x.Digest == y.Digest
})
}
// resolve is the provider requirement r of p points to by §708's order, or the hold that says why none is: none and no
// hold for an optional requirement nothing provides (ADR 0330 §3).
func resolve(p *placedAt, r string, placed []*placedAt, holders map[string]mma.SeatHolders, pins map[string]mma.Provider) (mma.Provider, *meshwants.Held) {
// 1. The named seat's holder, from the placements' holds as the state seat-holders keeps them; a delivering seat no
// placement holds holds the entry, and no other step is tried.
if def, ok := p.delivering[r]; ok {
node := ""
if def.Scope == seatdef.ScopeNode {
node = p.node
}
key := def.Seat
if node != "" {
key += "." + node
}
if sh := holders[key]; len(sh.Holders) > 0 {
return mma.Provider{Node: sh.Holders[0].Node, Module: sh.Holders[0].Module}, nil
}
return mma.Provider{}, &meshwants.Held{Requirement: r, Seat: def.Seat, Candidates: []meshwants.HeldCandidate{}}
}
// 2. A pin: the placed module's bindings, else its node's pins.
if prov, ok := p.placed.Bindings[r]; ok {
return prov, nil
}
if prov, ok := pins[r]; ok {
return prov, nil
}
// 3. The provider on the node; 4. the only provider.
var here, all []*placedAt
for _, q := range placed {
if q != p && slices.Contains(q.provides, r) {
all = append(all, q)
if q.node == p.node {
here = append(here, q)
}
}
}
switch {
case len(here) == 1:
return mma.Provider{Node: here[0].node, Module: here[0].placed.Module}, nil
case len(here) == 0 && len(all) == 1:
return mma.Provider{Node: all[0].node, Module: all[0].placed.Module}, nil
case len(all) == 0 && p.optional(r):
return mma.Provider{}, nil
}
h := &meshwants.Held{Requirement: r, Candidates: []meshwants.HeldCandidate{}}
for _, q := range all {
h.Candidates = append(h.Candidates, meshwants.HeldCandidate{Node: q.node, Module: q.placed.Module})
}
slices.SortFunc(h.Candidates, func(a, b meshwants.HeldCandidate) int {
return strings.Compare(a.Node+"/"+a.Module, b.Node+"/"+b.Module)
})
return mma.Provider{}, h
}
@@ -1,248 +0,0 @@
package assigngraph
import (
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Cycles are the cycles among the acting waits (design 47 G §272, ADR 0656, A-323): an entry waits for its acting parts in the step
// an edge names, so acting parts that wait for each other, in one step or across steps, wait for ever unless a rule breaks the
// cycle. A cycle within one step is `<step>: a -> b -> a`; one across steps names each edge's step, `a -[compose]-> b -[place]-> a`;
// each by node and module, said once from its smallest member, sorted. A holder's own inputs are no edge here (the graph gives an
// acting edge only from a part to the entries it acts on, never to what the part is built from).
func Cycles(entries []meshwants.AssignmentEntry) []string {
type key struct{ step, at string }
next := map[key][]string{}
steps := map[string]bool{}
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind != EdgeActing {
continue
}
k := key{edge.Step, e.Node + "/" + e.Module}
next[k] = append(next[k], edge.Node+"/"+edge.Module)
steps[edge.Step] = true
}
}
var out []string
for _, step := range slices.Sorted(maps.Keys(steps)) {
const (
open = iota + 1
done
)
state := map[string]int{}
var path []string
var walk func(at string)
walk = func(at string) {
state[at] = open
path = append(path, at)
for _, to := range next[key{step, at}] {
switch state[to] {
case 0:
walk(to)
case open:
from := slices.Index(path, to)
cycle := slices.Clone(path[from:])
// start at the smallest member so one cycle is said once, whichever member is met first
lowest := slices.Index(cycle, slices.Min(cycle))
cycle = append(cycle[lowest:], cycle[:lowest]...)
out = append(out, fmt.Sprintf("%s: %s -> %s", step, strings.Join(cycle, " -> "), cycle[0]))
}
}
path = path[:len(path)-1]
state[at] = done
}
starts := map[string]bool{}
for k := range next {
if k.step == step {
starts[k.at] = true
}
}
for _, at := range slices.Sorted(maps.Keys(starts)) {
if state[at] == 0 {
walk(at)
}
}
}
// Across steps (A-323): the shortest cycle through each entry over every acting edge, said when its edges name two steps or more.
for _, hops := range cyclesThrough(entries, func(_ meshwants.AssignmentEntry, e meshwants.Edge) bool { return e.Kind == EdgeActing }) {
if crossesSteps(hops) {
out = append(out, hopWords(rotated(hops)))
}
}
slices.Sort(out)
return slices.Compact(out)
}
// HeldCycles is, by `<node>/<module>`, the cycle each entry lies on among the edges holds says hold it (A-323, A-330): a want held
// on such a cycle is held for ever, since no rule breaks it. Each cycle starts at the entry and names each edge's step.
func HeldCycles(entries []meshwants.AssignmentEntry, holds func(e meshwants.AssignmentEntry, edge meshwants.Edge) bool) map[string]string {
out := map[string]string{}
for at, hops := range cyclesThrough(entries, holds) {
out[at] = hopWords(hops)
}
return out
}
// Engines is, by node, the module that holds the seat node-engine there, read from the acting edges' seat (A-330).
func Engines(entries []meshwants.AssignmentEntry) map[string]string {
out := map[string]string{}
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind == EdgeActing && edge.Seat == nodeengine.Seat {
out[edge.Node] = edge.Module
}
}
}
return out
}
// Runner is the module that holds the seat mesh-pipeline-runner, read from the acting edges' seat as Engines reads the node-engine
// (A-334); "" when none is read.
func Runner(entries []meshwants.AssignmentEntry) string {
for _, e := range entries {
for _, edge := range e.Edges {
if edge.Kind == EdgeActing && edge.Seat == meshpipelinerunner.Seat {
return edge.Module
}
}
}
return ""
}
// hop is one edge of a cycle: from the entry at, in step, to the next hop's entry.
type hop struct{ at, step string }
// cyclesThrough is, by `<node>/<module>`, the shortest cycle through each entry over the edges keep takes, found within the
// entry's strongly connected part, each starting at the entry.
func cyclesThrough(entries []meshwants.AssignmentEntry, keep func(meshwants.AssignmentEntry, meshwants.Edge) bool) map[string][]hop {
type out struct{ to, step string }
next := map[string][]out{}
for _, e := range entries {
at := e.Node + "/" + e.Module
for _, edge := range e.Edges {
if keep(e, edge) {
next[at] = append(next[at], out{edge.Node + "/" + edge.Module, edge.Step})
}
}
}
for at := range next {
slices.SortFunc(next[at], func(a, b out) int { return strings.Compare(a.to+"\x00"+a.step, b.to+"\x00"+b.step) })
}
part := strongParts(slices.Sorted(maps.Keys(next)), func(at string) []string {
var to []string
for _, o := range next[at] {
to = append(to, o.to)
}
return to
})
found := map[string][]hop{}
for _, start := range slices.Sorted(maps.Keys(next)) {
// breadth first from start back to it, within its part
prev := map[string]hop{}
queue := []string{start}
seen := map[string]bool{}
for len(queue) > 0 && found[start] == nil {
at := queue[0]
queue = queue[1:]
for _, o := range next[at] {
if part[o.to] != part[start] {
continue
}
if o.to == start {
hops := []hop{{at, o.step}}
for cur := at; cur != start; {
h := prev[cur]
hops = append(hops, h)
cur = h.at
}
slices.Reverse(hops)
found[start] = hops
break
}
if !seen[o.to] {
seen[o.to] = true
prev[o.to] = hop{at, o.step}
queue = append(queue, o.to)
}
}
}
}
return found
}
// strongParts numbers each vertex by its strongly connected part (Tarjan).
func strongParts(vertices []string, succ func(string) []string) map[string]int {
index, low, part := map[string]int{}, map[string]int{}, map[string]int{}
on := map[string]bool{}
var stack []string
n, parts := 0, 0
var visit func(v string)
visit = func(v string) {
index[v], low[v] = n, n
n++
stack = append(stack, v)
on[v] = true
for _, w := range succ(v) {
if _, ok := index[w]; !ok {
visit(w)
low[v] = min(low[v], low[w])
} else if on[w] {
low[v] = min(low[v], index[w])
}
}
if low[v] == index[v] {
parts++
for {
w := stack[len(stack)-1]
stack = stack[:len(stack)-1]
on[w] = false
part[w] = parts
if w == v {
break
}
}
}
}
for _, v := range vertices {
if _, ok := index[v]; !ok {
visit(v)
}
}
return part
}
func crossesSteps(hops []hop) bool {
for _, h := range hops[1:] {
if h.step != hops[0].step {
return true
}
}
return false
}
// rotated is the cycle started at its smallest member, so one cycle is said once.
func rotated(hops []hop) []hop {
lowest := 0
for i, h := range hops {
if h.at < hops[lowest].at {
lowest = i
}
}
return append(slices.Clone(hops[lowest:]), hops[:lowest]...)
}
// hopWords is a cycle with each edge's step: `a -[compose]-> b -[place]-> a`.
func hopWords(hops []hop) string {
var b strings.Builder
for _, h := range hops {
fmt.Fprintf(&b, "%s -[%s]-> ", h.at, h.step)
}
return b.String() + hops[0].at
}
@@ -1,60 +0,0 @@
package assigngraph
import (
"slices"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §272 (AS10), A-323: a cycle among the waits is found within one step and across steps, each edge's step named across
// steps; a chain and provisioning edges are none.
func TestACycleAmongTheWaitsWithinOrAcrossStepsIsFound(t *testing.T) {
act := func(node, module, step string) meshwants.Edge {
return meshwants.Edge{Kind: EdgeActing, Node: node, Module: module, Step: step}
}
entry := func(module string, edges ...meshwants.Edge) meshwants.AssignmentEntry {
return meshwants.AssignmentEntry{Node: "ace", Module: module, Edges: edges}
}
for _, c := range []struct {
name string
entries []meshwants.AssignmentEntry
want []string
}{
{"two parts on each other in one step", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "a", "compose"))},
[]string{"compose: ace/a -> ace/b -> ace/a"}},
{"three in a ring, said once", []meshwants.AssignmentEntry{entry("c", act("ace", "a", "build")), entry("a", act("ace", "b", "build")), entry("b", act("ace", "c", "build"))},
[]string{"build: ace/a -> ace/b -> ace/c -> ace/a"}},
{"the indexer taking in the checker, the checker checking the indexer", []meshwants.AssignmentEntry{entry("checker", act("ace", "indexer", "take-in")), entry("indexer", act("ace", "checker", "check"))},
[]string{"ace/checker -[take-in]-> ace/indexer -[check]-> ace/checker"}},
{"the assigner and the node-engine, across compose and place", []meshwants.AssignmentEntry{entry("assigner", act("ace", "engine", "compose")),
entry("engine", act("ace", "assigner", "place")), entry("app", act("ace", "engine", "compose"), act("ace", "assigner", "place"))},
[]string{"ace/assigner -[compose]-> ace/engine -[place]-> ace/assigner"}},
{"a chain", []meshwants.AssignmentEntry{entry("a", act("ace", "b", "compose")), entry("b", act("ace", "c", "compose")), entry("c")}, nil},
{"provisioning edges are no wait of a step", []meshwants.AssignmentEntry{entry("a", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "b"}), entry("b", meshwants.Edge{Kind: EdgeProvisioning, Node: "ace", Module: "a"})}, nil},
} {
if got := Cycles(c.entries); !slices.Equal(got, c.want) {
t.Errorf("%s: cycles %v, want %v", c.name, got, c.want)
}
}
}
// A-323, A-330: the node-engine on a node is read from the acting edges' seat, and the cycles that hold a want are those of the
// edges said to hold it, each from the entry, naming each edge's step.
func TestTheHeldCyclesAreThoseOfTheHoldingEdges(t *testing.T) {
entries := []meshwants.AssignmentEntry{
{Node: "ace", Module: "assigner", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "engine", Step: "compose", Seat: "node-engine"}}},
{Node: "ace", Module: "engine", Edges: []meshwants.Edge{{Kind: EdgeActing, Node: "ace", Module: "assigner", Step: "place", Seat: "mesh-wants"}}},
}
if got := Engines(entries); len(got) != 1 || got["ace"] != "engine" {
t.Errorf("the node-engines %v, want engine on ace", got)
}
all := func(meshwants.AssignmentEntry, meshwants.Edge) bool { return true }
if got := HeldCycles(entries, all); len(got) != 2 || got["ace/engine"] != "ace/engine -[place]-> ace/assigner -[compose]-> ace/engine" {
t.Errorf("held cycles %v", got)
}
// the node-engine's want held by no acting edge: no cycle holds anything
if got := HeldCycles(entries, func(e meshwants.AssignmentEntry, _ meshwants.Edge) bool { return e.Module != "engine" }); len(got) != 0 {
t.Errorf("held cycles with the node-engine's edges holding nothing: %v", got)
}
}
@@ -1,400 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
type memObjects map[string][]byte
func (m memObjects) Put(_ context.Context, name string, data []byte) error {
m[name] = data
return nil
}
func (m memObjects) Get(_ context.Context, name string) ([]byte, error) {
d, ok := m[name]
if !ok {
return nil, ErrNoObject
}
return d, nil
}
func (m memObjects) Names(context.Context) ([]string, error) {
out := []string{}
for n := range m {
out = append(out, n)
}
return out, nil
}
func half(branch string, revision int64, entries ...meshcatalog.CatalogEntry) meshcatalogclient.Half {
h := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Kind: "trunk", Branch: branch, Revision: revision, Source: revision + 10},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, e := range entries {
h.Entries[e.Module] = e
h.Revision.Entries = append(h.Revision.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: "sha256:" + e.Module + e.Fp})
}
return h
}
// Design 47 *Two graphs, one source of truth* (lines 1353-1366, 1416-1418) and *An acting part's new version acts first*
// (lines 1835-1836): each entry one module version on a node from the catalog graph of the branch its placement names,
// its edges provisioning and acting, a revision computed again from its inputs equal to the one kept.
func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) {
ctx := context.Background()
in := Inputs{
Halves: map[string]meshcatalogclient.Half{"main": half("main", 7,
meshcatalog.CatalogEntry{Module: "node-engine", Fp: "src2:e1", Acts: []meshcatalog.ActingRule{{Seat: "node-engine", Step: "compose", On: "every", Bound: "30m"}}},
meshcatalog.CatalogEntry{Module: "build-go", Fp: "src2:b1", Acts: []meshcatalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
meshcatalog.CatalogEntry{Module: "postgres", Fp: "src2:p1", Provides: []string{"database"}},
meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"database", "secrets"}, Toolchains: map[string]meshcatalog.BuildEdge{"go": {Version: "1"}}},
)},
Placements: map[string]mma.Assignments{
"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "postgres"}, {Module: "build-go"},
{Module: "web", Bindings: map[string]mma.Provider{"database": {Node: "anchor", Module: "postgres"}}}}},
"laptop": {Node: "laptop", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "web", Build: "src2:w0"},
{Module: "photos", Branch: "beta"}, {Module: "ghost"}}},
},
FirstBranch: func(module string) string {
if module == "ghost" {
return ""
}
return "main"
},
}
c := Compute(in)
byNode := map[string]meshwants.AssignmentEntry{}
for _, ref := range c.Revision.Entries {
byNode[ref.Node+"/"+ref.Module] = c.Entries[ref.Digest]
}
webA, webL := byNode["anchor/web"], byNode["laptop/web"]
if len(c.Revision.Entries) != 6 || len(c.Revision.Catalogs) != 1 || c.Revision.Catalogs[0] != (meshwants.CatalogOf{Branch: "main", Revision: 7, Source: 17}) {
t.Fatalf("revision %+v", c.Revision)
}
if webA.Branch != "main" || webA.Fp != "src2:w1" || webA.Catalog != "sha256:websrc2:w1" || webL.Fp != "src2:w0" {
t.Errorf("web's entries: %+v, %+v (the placement's build is the version on laptop)", webA, webL)
}
want := []meshwants.Edge{
{Kind: EdgeActing, Node: "anchor", Module: "build-go", Fp: "src2:b1", Seat: "node-build-go", Step: "build"},
{Kind: EdgeActing, Node: "anchor", Module: "node-engine", Fp: "src2:e1", Seat: "node-engine", Step: "compose", Bound: "30m"},
{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"},
}
if fmt.Sprint(webA.Edges) != fmt.Sprint(want) {
t.Errorf("web on anchor's edges:\n got %+v\nwant %+v", webA.Edges, want)
}
// laptop's web: its own node's node-engine composes it, the build module placed once builds it, and with no binding or
// pin its database is the only provider's (design 47 G §357).
if e := webL.Edges; len(e) != 3 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" ||
e[2] != (meshwants.Edge{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"}) {
t.Errorf("web on laptop's edges: %+v", e)
}
// Its secrets no placed module provides: held on both nodes, naming no candidate.
if h := webA.Held; len(h) != 1 || h[0].Requirement != "secrets" || len(h[0].Candidates) != 0 {
t.Errorf("web on anchor is held on %+v", h)
}
if e := byNode["anchor/postgres"].Edges; len(e) != 1 || e[0].Module != "node-engine" {
t.Errorf("postgres builds no Go, so only the node-engine acts on it: %+v", e)
}
if len(byNode["anchor/node-engine"].Edges) != 0 {
t.Errorf("an acting part acts on itself: %+v", byNode["anchor/node-engine"].Edges)
}
if strings.Join(c.Uncomputed, "|") != "laptop/ghost: no watched branch: its placement names none and no registration of it is known|"+
"laptop/photos: the catalog graph of the branch beta is not held" {
t.Errorf("uncomputed %q", c.Uncomputed)
}
// Kept once; computed again from the same inputs it is the same revision, kept no second time.
o := memObjects{}
at := time.Date(2026, 11, 10, 18, 0, 0, 0, time.UTC)
rev, kept, err := Keep(ctx, o, c, "catalog-revision", at)
if err != nil || !kept || rev.Revision != 1 || rev.Cause != "catalog-revision" || len(o) != 7 {
t.Fatalf("kept %v %+v (%v), objects %d", kept, rev, err, len(o))
}
if again, kept, err := Keep(ctx, o, Compute(in), "placement-changed", at); err != nil || kept || again.Revision != 1 {
t.Errorf("recomputed from the same inputs: kept %v, %+v, %v", kept, again, err)
}
// A placement change moves laptop's web to the catalog's version: revision 2, one entry more stored, the rest shared.
in.Placements["laptop"].Modules[1].Build = ""
rev2, kept, err := Keep(ctx, o, Compute(in), "placement-changed", at)
if err != nil || !kept || rev2.Revision != 2 || len(o) != 9 || rev2.Placements == rev.Placements {
t.Fatalf("revision 2 %v %+v (%v), objects %d", kept, rev2, err, len(o))
}
// The verbs read only what is kept.
r, err := Revision(ctx, o, meshwants.RevisionInput{})
if err != nil || r.Revision.Revision != 2 {
t.Errorf("revision = %+v, %v", r, err)
}
if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "g", Heads: "h"}); !errors.Is(err, ErrNoCandidate) {
t.Errorf("a candidate's revision: %v", err)
}
e, err := Entry(ctx, o, meshwants.EntryInput{Revision: 1, Node: "laptop", Module: "web"})
if err != nil || e.Entry.Fp != "src2:w0" || !strings.HasPrefix(e.Digest, "sha256:") {
t.Errorf("entry = %+v, %v", e, err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{From: 1, To: 2})
if err != nil || len(d.Changes) != 1 || fmt.Sprint(d.Changes[0]) != fmt.Sprint(meshwants.Change{Node: "laptop", Module: "web", From: "src2:w0", To: "src2:w1"}) ||
strings.Join(d.Nodes, ",") != "laptop" {
t.Errorf("difference = %+v, %v", d, err)
}
if _, err := Difference(ctx, o, meshwants.DifferenceInput{To: 2}); err == nil {
t.Error("a difference with one revision answered")
}
if ev := Event(rev2); ev.Revision != 2 || len(ev.Catalogs) != 1 || ev.Kind != "trunk" {
t.Errorf("event %+v", ev)
}
}
// Design 47 G §357, §708, §709 and A-167, A-229: a requirement points to the named seat's holder, else a pin, else the
// provider on the consumer's node, else the only provider; otherwise the entry is held, naming the candidates (and the
// unheld seat), and nothing is guessed. An optional requirement nothing provides is left absent; a requirement the
// manifest's contributes names gets a contribution edge to the same provider.
func TestARequirementPointsByTheOrderOrIsHeld(t *testing.T) {
for _, c := range []struct {
name string
kv []string // the nodes kv is placed on
pin bool // web's node pins kv on laptop
seat string // the seat delivering cache: "" none, "held" held by kv on server, "unheld" held by none
optional bool // web requires cache optionally
contributes bool // web's manifest contributes to cache
edge string // the node of the provisioning edge, "" for none
held string // the candidates' nodes of the hold, "-" for no hold
heldSeat string // the seat the hold names
}{
{name: "the provider on the consumer's node", kv: []string{"anchor", "laptop"}, edge: "anchor", held: "-"},
{name: "the only provider", kv: []string{"laptop"}, edge: "laptop", held: "-"},
{name: "a pin before the node's own", kv: []string{"anchor", "laptop"}, pin: true, edge: "laptop", held: "-"},
{name: "two providers elsewhere: held", kv: []string{"laptop", "server"}, held: "laptop,server"},
{name: "none: held naming none", held: ""},
{name: "the named seat's holder before the provider on the node", kv: []string{"anchor", "server"}, seat: "held", edge: "server", held: "-"},
{name: "a delivering seat no placement holds: held naming it", kv: []string{"anchor"}, seat: "unheld", held: "", heldSeat: "cache-seat"},
{name: "an optional requirement nothing provides: absent", optional: true, held: "-"},
{name: "an optional requirement with two providers elsewhere: held", optional: true, kv: []string{"laptop", "server"}, held: "laptop,server"},
{name: "a contributed requirement: a contribution edge too", kv: []string{"laptop"}, contributes: true, edge: "laptop", held: "-"},
} {
t.Run(c.name, func(t *testing.T) {
web := meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"cache"},
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"cache"}}}
if c.optional {
web.Manifest["requires"] = []any{map[string]any{"name": "cache", "optional": true}}
}
if c.contributes {
web.Contributes = []string{"cache"}
}
h := half("main", 3, meshcatalog.CatalogEntry{Module: "kv", Fp: "src2:k1", Provides: []string{"cache"}}, web)
defs := map[string]seatdef.Definition{}
if c.seat != "" {
h.Revision.Definitions = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "cache-seat", Digest: "sha256:d"}}
defs["cache-seat"] = seatdef.Definition{Seat: "cache-seat", Scope: seatdef.ScopeMesh, Delivers: "cache"}
}
pl := map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}}
if c.pin {
pl["anchor"] = mma.Assignments{Node: "anchor", Modules: pl["anchor"].Modules, Pins: map[string]mma.Provider{"cache": {Node: "laptop", Module: "kv"}}}
}
for _, n := range c.kv {
a := pl[n]
a.Node, a.Modules = n, append(a.Modules, mma.PlacedModule{Module: "kv"})
pl[n] = a
}
if c.seat == "held" {
a := pl["server"]
a.Holds = append(a.Holds, mma.Holding{Claim: "cache-seat", Module: "kv"})
pl["server"] = a
}
out := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: pl, FirstBranch: func(string) string { return "main" },
Definitions: defs})
var got meshwants.AssignmentEntry
for _, ref := range out.Revision.Entries {
if ref.Node == "anchor" && ref.Module == "web" {
got = out.Entries[ref.Digest]
}
}
edge, contribution := "", ""
for _, e := range got.Edges {
if e.Kind == EdgeProvisioning && e.Provision == "cache" {
edge = e.Node
}
if e.Kind == EdgeContribution && e.Provision == "cache" {
contribution = e.Node + "/" + e.Module
}
}
held, heldSeat := "-", ""
if len(got.Held) == 1 && got.Held[0].Requirement == "cache" {
var nodes []string
for _, cand := range got.Held[0].Candidates {
nodes = append(nodes, cand.Node)
}
held, heldSeat = strings.Join(nodes, ","), got.Held[0].Seat
}
if edge != c.edge || held != c.held || heldSeat != c.heldSeat || len(got.Held) > 1 {
t.Errorf("web points to %q, held on %q seat %q (%+v); want %q, %q, %q", edge, held, heldSeat, got.Held, c.edge, c.held, c.heldSeat)
}
if want := map[bool]string{true: c.edge + "/kv"}[c.contributes]; contribution != want {
t.Errorf("web's contribution edge points to %q, want %q", contribution, want)
}
})
}
}
// Design 47 line 1385: a whole assignment graph revision of about 120 modules on four nodes is under the bus's 256 KiB
// value limit.
func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) {
var entries []meshcatalog.CatalogEntry
placements := map[string]mma.Assignments{}
for i := range 120 {
entries = append(entries, meshcatalog.CatalogEntry{Module: fmt.Sprintf("module-%03d", i), Fp: "src2:" + strings.Repeat("a", 64)})
}
for _, node := range []string{"anchor", "home", "desk", "laptop"} {
a := mma.Assignments{Node: node}
for _, e := range entries {
a.Modules = append(a.Modules, mma.PlacedModule{Module: e.Module})
}
placements[node] = a
}
h := half("main", 1, entries...)
for i := range h.Revision.Entries {
h.Revision.Entries[i].Digest = "sha256:" + strings.Repeat("b", 64)
}
c := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: placements, FirstBranch: func(string) string { return "main" }})
raw, _ := json.Marshal(c.Revision)
if len(c.Revision.Entries) != 480 || len(raw) >= 256<<10 {
t.Errorf("%d entries in %d bytes", len(c.Revision.Entries), len(raw))
}
}
func (m memObjects) Delete(_ context.Context, name string) error {
delete(m, name)
return nil
}
// A-201, A-235, §186, §189: a run's candidate assignment graph is kept apart from trunk's under candidate.<group>.<heads>, served by
// revision, entry and difference against trunk, and dropped by its group alone.
func TestARunsCandidateGraphIsKeptApartServedAndDropped(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
placed := map[string]mma.Assignments{"ace": {Modules: []mma.PlacedModule{{Module: "web"}}}}
first := func(string) string { return "main" }
o := memObjects{}
trunk := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 3, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1"})},
Placements: placed, FirstBranch: first})
if _, _, err := Keep(ctx, o, trunk, "catalog-revision", at); err != nil {
t.Fatal(err)
}
cand := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 4, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w2"})},
Placements: placed, FirstBranch: first})
if _, _, err := KeepCandidate(ctx, o, cand, "", "h1", "catalog-revision", at); err == nil {
t.Error("a candidate naming no group was kept")
}
rev, fresh, err := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at)
if err != nil || !fresh || rev.Kind != meshcatalogclient.KindCandidate || rev.Group != "run-7" || rev.Revision != 1 {
t.Fatalf("kept %+v fresh %v: %v", rev, fresh, err)
}
if _, ok := o[CandidateObject("run-7", "h1")]; !ok {
t.Fatalf("the candidate is not under candidate.run-7.h1: %v", o)
}
if _, fresh, _ := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at); fresh {
t.Error("the same candidate kept again is a new revision")
}
newest, has, err := Newest(ctx, o)
if err != nil || !has || newest.Kind != meshcatalogclient.KindTrunk || newest.Revision != 1 {
t.Errorf("trunk's newest is %+v (%v, %v): the candidate is not kept apart", newest, has, err)
}
for _, in := range []meshwants.RevisionInput{{Group: "run-7"}, {Group: "run-7", Heads: "h1"}} {
a, err := Revision(ctx, o, in)
if err != nil || a.Revision.Group != "run-7" {
t.Errorf("revision %+v answered %+v: %v", in, a.Revision, err)
}
}
if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "run-8"}); !errors.Is(err, ErrNoCandidate) {
t.Errorf("an unkept candidate answered %v", err)
}
e, err := Entry(ctx, o, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"})
if err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the candidate's entry is %+v: %v", e.Entry, err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{Group: "run-7"})
if err != nil || len(d.Changes) != 1 || d.Changes[0].From != "src2:w1" || d.Changes[0].To != "src2:w2" {
t.Errorf("the candidate against trunk is %+v: %v", d, err)
}
if _, err := Difference(ctx, o, meshwants.DifferenceInput{}); err == nil {
t.Error("difference with none of from, to and group is not refused")
}
if _, _, err := KeepCandidate(ctx, o, cand, "run-9", "h2", "catalog-revision", at); err != nil {
t.Fatal(err)
}
if n, err := DropCandidate(ctx, o, "run-7"); err != nil || n != 1 {
t.Errorf("dropped %d: %v", n, err)
}
if _, err := ReadCandidate(ctx, o, "run-7", ""); !errors.Is(err, ErrNoCandidate) {
t.Errorf("the dropped candidate reads: %v", err)
}
if _, err := ReadCandidate(ctx, o, "run-9", ""); err != nil {
t.Errorf("another run's candidate went with it: %v", err)
}
if _, has, _ := Newest(ctx, o); !has {
t.Error("dropping a candidate dropped trunk")
}
}
// Design 47 G §512, §186, A-242: the difference orders its changes itself: each change names the changes it is ordered after
// with the kind of the edge, and Tiers holds the changes in that order, an acting part and a provider before what they act on.
func TestTheDifferenceFillsItsTiersAndEachChangesAfter(t *testing.T) {
ctx := context.Background()
build := func(ne, pg, web string) Computed {
return Compute(Inputs{
Halves: map[string]meshcatalogclient.Half{"main": half("main", 7,
meshcatalog.CatalogEntry{Module: "node-engine", Fp: ne, Acts: []meshcatalog.ActingRule{{Seat: "node-engine", Step: "compose", On: "every"}}},
meshcatalog.CatalogEntry{Module: "postgres", Fp: pg, Provides: []string{"database"}},
meshcatalog.CatalogEntry{Module: "web", Fp: web, Requires: []string{"database"}},
)},
Placements: map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "node-engine"}, {Module: "postgres"},
{Module: "web", Bindings: map[string]mma.Provider{"database": {Node: "anchor", Module: "postgres"}}}}}},
FirstBranch: func(string) string { return "main" },
})
}
o := memObjects{}
at := time.Date(2026, 11, 12, 1, 0, 0, 0, time.UTC)
if _, _, err := Keep(ctx, o, build("src2:e1", "src2:p1", "src2:w1"), "catalog-revision", at); err != nil {
t.Fatal(err)
}
if _, _, err := Keep(ctx, o, build("src2:e2", "src2:p2", "src2:w2"), "catalog-revision", at); err != nil {
t.Fatal(err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{From: 1, To: 2})
if err != nil || len(d.Changes) != 3 {
t.Fatalf("difference %+v, %v", d, err)
}
after := map[string]string{}
for _, c := range d.Changes {
var a []string
for _, x := range c.After {
a = append(a, x.Kind+":"+x.Node+"/"+x.Module)
}
after[c.Module] = strings.Join(a, ",")
}
if after["node-engine"] != "" || after["postgres"] != "acting:anchor/node-engine" ||
after["web"] != "acting:anchor/node-engine,provisioning:anchor/postgres" {
t.Errorf("after = %v", after)
}
var tiers []string
for _, tier := range d.Tiers {
var ms []string
for _, r := range tier {
ms = append(ms, r.Module)
}
tiers = append(tiers, strings.Join(ms, ","))
}
if strings.Join(tiers, "|") != "node-engine|postgres|web" {
t.Errorf("tiers = %v", tiers)
}
}
@@ -1,185 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// RevisionsObjects is the local name of the kept data in the assigner's manifest's old `objects` section (design 47
// lines 1362-1366: both foundation parts keep their revisions as objects in the old form; the seat mesh-wants' state
// assignment-revisions names its key forms).
const RevisionsObjects = "assignment-revisions"
// ErrNoObject is an object the bucket does not hold.
var ErrNoObject = errors.New("no such object")
// Objects is an object bucket as the assigner keeps its revisions in it.
type Objects interface {
// Put stores data whole under name, replacing one of that name.
Put(ctx context.Context, name string, data []byte) error
// Get is the object's bytes; ErrNoObject when it holds none of that name.
Get(ctx context.Context, name string) ([]byte, error)
// Names is every object's name; none is an empty list.
Names(ctx context.Context) ([]string, error)
}
// EntryObject is the object name of the entry of that digest (the seat's key form entry.<digest>).
func EntryObject(digest string) string { return "entry." + digest }
// TrunkObject is the object name of a trunk revision (the seat's key form trunk.<revision>).
func TrunkObject(revision int64) string { return "trunk." + strconv.FormatInt(revision, 10) }
// trunkOf is the revision number a trunk object's name holds, and whether it is one.
func trunkOf(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// Newest is the newest trunk revision kept, its list only; none kept is false.
func Newest(ctx context.Context, o Objects) (meshwants.AssignmentRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var newest int64
for _, n := range names {
if r, ok := trunkOf(n); ok && r > newest {
newest = r
}
}
if newest == 0 {
return meshwants.AssignmentRevision{}, false, nil
}
rev, err := ReadRevision(ctx, o, newest)
return rev, err == nil, err
}
// ReadRevision is the trunk revision of that number, its list only.
func ReadRevision(ctx context.Context, o Objects, revision int64) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, TrunkObject(revision))
if err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
return rev, nil
}
// ReadEntry is the entry kept under that digest.
func ReadEntry(ctx context.Context, o Objects, digest string) (meshwants.AssignmentEntry, error) {
var e meshwants.AssignmentEntry
data, err := o.Get(ctx, EntryObject(digest))
if err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
if err := json.Unmarshal(data, &e); err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
return e, nil
}
// Keep keeps c as the next trunk revision, numbered after the newest kept, unless the newest kept lists the same entries
// from the same inputs (Same): then it keeps nothing and answers the newest and false. Each entry object not held yet is
// written before the list, so a list is never kept whose entries are not, and an entry two revisions share is stored
// once (design 47 *An unchanged entry is stored once*).
func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
newest, has, err := Newest(ctx, o)
if err != nil {
return newest, false, err
}
if has && Same(newest, c.Revision) {
return newest, false, nil
}
rev := c.Revision
rev.Revision = newest.Revision + 1
rev.Cause = cause
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return newest, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return newest, false, err
}
if err := o.Put(ctx, TrunkObject(rev.Revision), list); err != nil {
return newest, false, fmt.Errorf("revision %d in %s: %w", rev.Revision, RevisionsObjects, err)
}
return rev, true, nil
}
// writeEntries writes each entry object rev lists that the bucket does not hold yet, from c, before its list is kept.
func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.AssignmentRevision) error {
names, err := o.Names(ctx)
if err != nil {
return fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if held[name] {
continue
}
e, ok := c.Entries[ref.Digest]
if !ok {
return fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest)
}
data, err := json.Marshal(e)
if err != nil {
return err
}
if err := o.Put(ctx, name, data); err != nil {
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
}
held[name] = true
}
return nil
}
// Event is the seat's event assignment-revision of a kept revision.
func Event(rev meshwants.AssignmentRevision) meshwants.AssignmentRevisionEvent {
return meshwants.AssignmentRevisionEvent{V: rev.V, Revision: rev.Revision, Kind: rev.Kind, Cause: rev.Cause, Group: rev.Group,
Heads: rev.Heads, Catalogs: rev.Catalogs, At: rev.At}
}
// Filler is an object bucket that says its fill: the bytes used and its cap in bytes (0: no cap known).
type Filler interface {
Fill(ctx context.Context) (used, capacity int64, err error)
}
// NewBytes is what keeping c next would add to the bucket: each entry object not held yet and the list (design 47 G §181).
func NewBytes(ctx context.Context, o Objects, c Computed) int64 {
names, err := o.Names(ctx)
if err != nil {
return 0
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
var size int64
for _, ref := range c.Revision.Entries {
if e, ok := c.Entries[ref.Digest]; ok && !held[EntryObject(ref.Digest)] {
data, _ := json.Marshal(e)
size += int64(len(data))
}
}
list, _ := json.Marshal(c.Revision)
return size + int64(len(list))
}
@@ -1,6 +0,0 @@
package assigngraph
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
@@ -1,81 +0,0 @@
package assigngraph
import (
"context"
"encoding/json"
"fmt"
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Reverted says a want of module on node at wantFp is a revert (design 47 G §298, §300, A-217): the module had wantFp in an earlier
// assignment graph revision than the one it now runs from, by the revisions' own numbers. The revision it runs from is the first kept
// that lists runFp for the module on the node; a want is a revert when it was listed before that, and never listed since.
func Reverted(ctx context.Context, o Objects, node, module, runFp, wantFp string) (bool, error) {
if runFp == "" || wantFp == "" || runFp == wantFp {
return false, nil
}
names, err := o.Names(ctx)
if err != nil {
return false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var numbers []int64
for _, n := range names {
if r, ok := trunkOf(n); ok {
numbers = append(numbers, r)
}
}
slices.Sort(numbers)
var ranFrom, lastWanted int64
for _, r := range numbers {
rev, err := ReadRevision(ctx, o, r)
if err != nil {
return false, err
}
for _, ref := range rev.Entries {
if ref.Node != node || ref.Module != module {
continue
}
e, err := ReadEntry(ctx, o, ref.Digest)
if err != nil {
return false, err
}
switch e.Fp {
case runFp:
if ranFrom == 0 {
ranFrom = r
}
case wantFp:
lastWanted = r
}
}
}
return ranFrom != 0 && lastWanted != 0 && lastWanted < ranFrom, nil
}
// FormChanges are the kept data whose `form` differs between the running entry's manifest and the want's (design 47 G §295 to §298,
// A-217): the entry's manifest whole is read as a manifest and compared by catalogue.FormChanges, so a kept data's form is read from
// the entry's kept declarations (catalogue.KeptDeclarations), never from a field of the want.
func FormChanges(running, applied meshcatalog.CatalogEntry) ([]catalogue.FormChange, error) {
before, err := manifestOf(running)
if err != nil {
return nil, fmt.Errorf("the running manifest of %s: %w", running.Module, err)
}
after, err := manifestOf(applied)
if err != nil {
return nil, fmt.Errorf("the manifest of %s: %w", applied.Module, err)
}
return catalogue.FormChanges(before, after), nil
}
func manifestOf(e meshcatalog.CatalogEntry) (catalogue.Manifest, error) {
var m catalogue.Manifest
raw, err := json.Marshal(e.Manifest)
if err != nil {
return m, err
}
err = json.Unmarshal(raw, &m)
return m, err
}

Some files were not shown because too many files have changed in this diff Show More