Compare commits

..
Author SHA1 Message Date
jschoubben 3ae3519714 The assigner pins mesh-sdk at c77d17f: a follower joining a watched state loses no change
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:52:33 +01:00
jschoubben 04aa1a1ace mesh-grants and the builder pin mesh-sdk at 8311ecb
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:36:58 +01:00
jschoubben b3a74e0bdc Merge builderA3's upgrade compare and builderA2's watched reader into group A 2026-11-01 03:35:05 +01:00
jschoubben b8d4a77e75 mesh-wants upgrade reads through mesh-sdk's watched reader, keeping no mirror of its own
One helper serves every part that reads whole states (ADR 0580 clause 3): the assigner's own watched mirror goes, its
compare reads the large states from stdio.WatchedStates and each manifest once per digest. mesh-sdk pinned at 8311ecb.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:34:21 +01:00
jschoubben 8112fe4b13 mesh-grants and the assigner read the grants part's key alone, root-free is each engine's word, and the hand-act log is the seat's alone (ADR 0571 clauses 9 and 11, ADR 0565 clause 11, issues 1953 and 1955)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:26:59 +01:00
jschoubben b74c371ed6 The builder asks the artifact sweep's seat for held and no longer the controller (ADR 0571 clause 2, issue 1955)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:26:59 +01:00
jschoubben f95d5b4d23 mesh-wants upgrade: read the large states by one watch each, started once, not a get per key, so compare answers in time (issue 1953 L5f)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:22:12 +01:00
1736 changed files with 44714 additions and 254811 deletions
-6
View File
@@ -2,9 +2,3 @@ vendor/
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
modules/*/cmd/*/mesh-*
!modules/*/cmd/*/mesh-*.go
# `go build .` at a module's root leaves its program there too (the view's, met a second time).
modules/*/mesh-*
# The delivery SDK's commands (design 47 G §464), built in their own directory or at the SDK's root.
/modules/mesh-delivery-sdk/cmd/seatgen/seatgen
/modules/mesh-delivery-sdk/seatgen/seatgen
/modules/mesh-delivery-sdk/mesh-pack
+7
View File
@@ -0,0 +1,7 @@
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
-293
View File
@@ -1,293 +0,0 @@
// Package asks holds the layout of the assigner's state `asks`: the keys of the asks the assigner puts to a person and the
// value it keeps at each, which the assigner writes and the view and the teller read (novox/hq ADR 0634 §1, ADR 0635 §5). It
// is the assigner's own published package. The gate's merge ask (`merge.<group>.<heads hash>`) needs proof of who answered
// (ADR 0421 §1), so it is answered on the phone alone until issue 953 closes. The assigner's asks need no such proof, and the
// view may answer them: the hold of a fingerprint whose health check failed (`hold.<module>.<fingerprint hex>`: release it or
// keep it held), the restore of a data module set back (`restore.<node>.<module>.<fingerprint hex>`: restore and lose newer
// writes, or leave it stopped, ADR 0429 §6), a backup that failed before an apply, a move a person must allow for a module of
// policy record, and the authorising ask of a proposal of assignments.
package asks
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/kept"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const (
// AskKindHold is the kind of the assigner's ask about a held fingerprint.
AskKindHold = "hold"
// AskKindRestore is the kind of the assigner's ask about a data module set back.
AskKindRestore = "restore"
// AskKindBackup is the kind of the assigner's ask about a backup before an apply that failed: back up again, or go on
// without one (novox/hq ADR 0539 clause 16).
AskKindBackup = "backup"
// AskKindRecord is the kind of the assigner's ask about a move a person must allow, for a module of policy record: a
// composition judged to recreate a container or move data, or one that cannot be judged (ADR 0539 clause 14).
AskKindRecord = "record"
// AskKindAssignments is the kind of the authorising ask about a proposal of assignments (novox/hq ADR 0567 §5):
// `assignments.<proposal id>`. The kind needs no proof, as the bus's put does not: its yes is an Approve, which only a
// channel proving who answered takes and the view never does (ViewAnswerFits).
AskKindAssignments = "assignments"
// AskKindOutside is the kind of the assigner's ask about a moved module whose failing health check has waited past its bound
// on an outside cause (design 47 G §706): go on, or count it as failed. `outside.<node>.<module>.<fingerprint hex>`.
AskKindOutside = "outside"
// AskKindRevert is the kind of the assigner's ask about a held revert, one with no restore point (design 47 G §710): keep it
// held, or start the reverted build on the live data. `revert.<node>.<module>.<fingerprint hex>`, the reverted build's.
AskKindRevert = "revert"
)
// AskKey is an ask's key read whole: its Kind, and the parts its kind has: a merge's Group and heads Hash, a hold's Module
// and fingerprint FP, a restore's, a backup's, an outside's and a revert's Node, Module and FP, a record's Node, Module and the digest Composed of the
// composition it would allow, the assignments' Proposal.
type AskKey struct {
Proposal string
Kind string
Group string
Hash string
Node string
Module string
FP string
Composed string
}
// AskNeedsProof says whether an ask of this kind must prove who answered it, so the view may not answer it: the merge's
// Approve does (ADR 0421 §1, issue 953), the assigner's hold and restore do not. An unknown kind is refused.
func AskNeedsProof(kind string) (bool, error) {
switch kind {
case meshdeliveryclient.AskKindMerge:
return true, nil
case AskKindHold, AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert:
return false, nil
}
return false, fmt.Errorf("asks: %q is not a kind of ask, %s, %s, %s, %s, %s, %s, %s or %s", kind, meshdeliveryclient.AskKindMerge, AskKindHold,
AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert)
}
// HoldAskKey is the key of the ask about a module's held fingerprint: `hold.<module>.<hex>`.
func HoldAskKey(module, fingerprint string) (string, error) {
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindHold + "." + key, nil
}
// RestoreAskKey is the key of the ask about a data module set back on a node from a fingerprint:
// `restore.<node>.<module>.<hex>`.
func RestoreAskKey(node, module, fingerprint string) (string, error) {
if err := keys.CheckName("node", node); err != nil {
return "", err
}
key, err := builds.BuilderBuildKey(module, fingerprint)
if err != nil {
return "", err
}
return AskKindRestore + "." + node + "." + key, nil
}
// OutsideAskKey is the key of the ask about a module's failing check on a node that waited past its bound on an outside cause:
// `outside.<node>.<module>.<hex>`.
func OutsideAskKey(node, module, fingerprint string) (string, error) {
key, err := RestoreAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindOutside + strings.TrimPrefix(key, AskKindRestore), nil
}
// OutsideAbout is what the outside ask is about: `<node>.<module>.<hex>` (design 47 G §706).
func OutsideAbout(key string) string { return strings.TrimPrefix(key, AskKindOutside+".") }
// BackupAskKey is the key of the ask about a module's failed backup on a node before it applies a fingerprint:
// `backup.<node>.<module>.<hex>`.
func BackupAskKey(node, module, fingerprint string) (string, error) {
part, err := catalogkey.KeyPart(fingerprint)
if err != nil {
return "", err
}
key, err := nodeengineclient.ComposedKey(node, module, part)
if err != nil {
return "", err
}
return AskKindBackup + "." + key, nil
}
// RevertAskKey is the key of the ask about a held revert of a module on a node to the reverted build's fingerprint:
// `revert.<node>.<module>.<hex>` (design 47 G §710).
func RevertAskKey(node, module, fingerprint string) (string, error) {
key, err := BackupAskKey(node, module, fingerprint)
if err != nil {
return "", err
}
return AskKindRevert + strings.TrimPrefix(key, AskKindBackup), nil
}
// AssignmentsAskKey is the key of the authorising ask about a proposal of assignments: `assignments.<proposal id>` (ADR 0567 §5).
func AssignmentsAskKey(proposal string) (string, error) {
if err := meshmoduleassignerclient.CheckProposalID(proposal); err != nil {
return "", err
}
return AskKindAssignments + "." + proposal, nil
}
// RecordAskKey is the key of the ask about a move of a module on a node to the composition whose digest it names:
// `record.<node>.<module>.<digest hex>`, so a new revision at the same fingerprint is asked again (ADR 0539 clause 14).
func RecordAskKey(node, module, composed string) (string, error) {
have, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return "", err
}
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return "", err
}
return AskKindRecord + "." + have + "." + strings.TrimPrefix(composed, "sha256:"), nil
}
// ParseAnyAskKey reads an ask key of any kind (a merge, hold, restore, backup, record, assignments, outside or revert ask); it refuses an unknown
// kind and a key whose parts do not fit its kind, naming the key.
func ParseAnyAskKey(key string) (AskKey, error) {
kind, rest, _ := strings.Cut(key, ".")
refused := func(err error) (AskKey, error) {
return AskKey{}, fmt.Errorf("asks: %q is not an ask key: %w", key, err)
}
switch kind {
case meshdeliveryclient.AskKindMerge:
_, group, hash, err := meshdeliveryclient.ParseAskKey(key)
if err != nil {
return AskKey{}, err
}
return AskKey{Kind: kind, Group: group, Hash: hash}, nil
case AskKindHold:
module, fp, err := builds.ParseBuilderBuildKey(rest)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Module: module, FP: fp}, nil
case AskKindRestore:
node, build, ok := strings.Cut(rest, ".")
if !ok {
return refused(fmt.Errorf("a restore ask is restore.<node>.<module>.<fingerprint hex>"))
}
if err := keys.CheckName("node", node); err != nil {
return refused(err)
}
module, fp, err := builds.ParseBuilderBuildKey(build)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindOutside:
k, err := ParseAnyAskKey(AskKindRestore + "." + rest)
if err != nil {
return refused(fmt.Errorf("an outside ask is outside.<node>.<module>.<fingerprint hex>"))
}
k.Kind = kind
return k, nil
case AskKindBackup, AskKindRevert:
node, module, part, isNode, err := nodeengineclient.ParseComposedKey(rest)
if err != nil || isNode {
return refused(fmt.Errorf("a %s ask is %s.<node>.<module>.<fingerprint hex>", kind, kind))
}
fp, err := catalogkey.ParseKeyPart(part)
if err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
case AskKindAssignments:
if err := meshmoduleassignerclient.CheckProposalID(rest); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Proposal: rest}, nil
case AskKindRecord:
parts := strings.Split(rest, ".")
if len(parts) != 3 {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
node, module, isNode, err := nodeengineclient.ParseHaveKey(parts[0] + "." + parts[1])
if err != nil || isNode {
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
}
composed := "sha256:" + parts[2]
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
return refused(err)
}
return AskKey{Kind: kind, Node: node, Module: module, Composed: composed}, nil
}
_, err := AskNeedsProof(kind)
return refused(err)
}
// Ask is the value of a key in the assigner's state asks: the router's ask whole, as the assigner composes it, so the teller
// publishes it unchanged. Asking counts the times the assigner wrote the ask at this key, as the gate's ask does: 0 or 1 for
// the first, n for the n-th, and the ask's id is meshdeliveryclient.AskingID(key, Asking), so each asking has its own id and
// an answer to an earlier asking is never taken for a later one. The ask is the operator-channel seat's as its JSON: its id,
// its options and its digest are checked by the assigner and the teller through operator-channel's client (novox/hq ADR
// 0627 §4).
type Ask struct {
V int `json:"v"`
Ask json.RawMessage `json:"ask"`
Asking int `json:"asking,omitempty"`
}
func (a Ask) check(key string) error {
ask, err := ParseAnyAskKey(key)
if err != nil {
return err
}
if proof, _ := AskNeedsProof(ask.Kind); proof {
return fmt.Errorf("asks: the assigner asks no %s ask: it needs proof of who answered", ask.Kind)
}
if a.Asking < 0 {
return fmt.Errorf("asks: the ask's asking %d is negative", a.Asking)
}
var fields map[string]json.RawMessage
if len(bytes.TrimSpace(a.Ask)) == 0 || json.Unmarshal(a.Ask, &fields) != nil || fields == nil {
return fmt.Errorf("asks: the value carries no router's ask")
}
return nil
}
// EncodeAsk is a, the ask at key, as JSON at keys.Version; it refuses a key of a kind that needs proof or of no kind, a
// negative asking, and a value carrying no router's ask.
func EncodeAsk(key string, a Ask) ([]byte, error) {
if err := a.check(key); err != nil {
return nil, err
}
a.V = keys.Version
return json.Marshal(a)
}
// DecodeAsk reads the ask at key; it refuses an unknown version (keys.ErrUnknownVersion) and what EncodeAsk refuses.
func DecodeAsk(key string, data []byte) (Ask, error) {
var a Ask
if err := kept.Decode(data, &a); err != nil {
return Ask{}, err
}
if err := a.check(key); err != nil {
return Ask{}, err
}
return a, nil
}
// SameAsk says whether two asks are the same value: the same version, asking and router's ask, its JSON compared compacted.
func SameAsk(a, b Ask) bool {
return a.V == b.V && a.Asking == b.Asking && bytes.Equal(compacted(a.Ask), compacted(b.Ask))
}
func compacted(raw json.RawMessage) []byte {
var b bytes.Buffer
if json.Compact(&b, raw) != nil {
return raw
}
return b.Bytes()
}
@@ -1,89 +0,0 @@
package asks
import (
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The keys equal what the assigner has always written (the expected strings are the trunk's own output for the same input),
// and each reads back to its parts.
func TestTheAssignersAskKeysKeepTheirFormsAndTheValueItsRules(t *testing.T) {
fp := "src2:" + strings.Repeat("ab", 32)
hex := strings.Repeat("ab", 32)
digest := "sha256:" + strings.Repeat("cd", 32)
hold, _ := HoldAskKey("gitea", fp)
restore, _ := RestoreAskKey("n1", "gitea", fp)
backup, _ := BackupAskKey("n1", "novox.be", fp)
assignments, _ := AssignmentsAskKey("p-20261028t183000-1a2b3c")
record, _ := RecordAskKey("n1", "gitea", digest)
revert, _ := RevertAskKey("n1", "nats", fp)
for key, want := range map[string]struct {
got string
back AskKey
}{
"hold": {hold, AskKey{Kind: "hold", Module: "gitea", FP: fp}},
"restore": {restore, AskKey{Kind: "restore", Node: "n1", Module: "gitea", FP: fp}},
"backup": {backup, AskKey{Kind: "backup", Node: "n1", Module: "novox.be", FP: fp}},
"assignments": {assignments, AskKey{Kind: "assignments", Proposal: "p-20261028t183000-1a2b3c"}},
"record": {record, AskKey{Kind: "record", Node: "n1", Module: "gitea", Composed: digest}},
"revert": {revert, AskKey{Kind: "revert", Node: "n1", Module: "nats", FP: fp}},
} {
back, err := ParseAnyAskKey(want.got)
if err != nil || back != want.back {
t.Errorf("%s: ParseAnyAskKey(%q) = %+v, %v; want %+v", key, want.got, back, err, want.back)
}
}
for got, want := range map[string]string{hold: "hold.gitea." + hex, restore: "restore.n1.gitea." + hex, backup: "backup.n1.novox=2ebe." + hex,
assignments: "assignments.p-20261028t183000-1a2b3c", record: "record.n1.gitea." + strings.Repeat("cd", 32), revert: "revert.n1.nats." + hex} {
if got != want {
t.Errorf("key %q, want %q", got, want)
}
}
merge, err := ParseAnyAskKey("merge.fix/1=2e2.d106c7950d28")
if err != nil || merge.Kind != "merge" || merge.Group != "fix/1.2" || merge.Hash != "d106c7950d28" {
t.Errorf("a merge ask key: %+v, %v", merge, err)
}
for _, bad := range []string{"", "other.x", "hold.gitea", "hold.gitea.src2:" + hex, "restore.n1", "backup.n1.node", "backup.n1.gitea." + hex[:60],
"assignment.AB", "record.n1.gitea." + hex[:10], "record.n1.node." + hex, "merge.g", "revert.n1.node", "revert.n1.nats." + hex[:60]} {
if _, err := ParseAnyAskKey(bad); err == nil {
t.Errorf("ParseAnyAskKey(%q) reads a wrong key", bad)
}
}
if proof, err := AskNeedsProof("merge"); err != nil || !proof {
t.Error("a merge ask needs proof")
}
for _, kind := range []string{"hold", "revert"} {
if proof, err := AskNeedsProof(kind); err != nil || proof {
t.Errorf("a %s ask needs no proof", kind)
}
}
if _, err := AskNeedsProof("view"); err == nil {
t.Error("an unknown kind of ask is taken")
}
// The value: the router's ask whole; an ask of a kind that needs proof, a negative asking and no ask are refused.
good := Ask{Ask: []byte(`{"id":"x"}`), Asking: 2}
data, err := EncodeAsk(hold, good)
if err != nil {
t.Fatalf("EncodeAsk: %v", err)
}
back, err := DecodeAsk(hold, data)
if err != nil || !SameAsk(back, Ask{V: keys.Version, Ask: []byte(` {"id": "x"} `), Asking: 2}) {
t.Errorf("DecodeAsk = %+v, %v", back, err)
}
for name, c := range map[string]struct {
key string
a Ask
}{"a merge ask": {"merge.g.d106c7950d28", good}, "a negative asking": {hold, Ask{Ask: good.Ask, Asking: -1}}, "no ask": {hold, Ask{}},
"a null ask": {hold, Ask{Ask: []byte("null")}}, "an unknown kind": {"other.x", good}} {
if _, err := EncodeAsk(c.key, c.a); err == nil {
t.Errorf("EncodeAsk takes %s", name)
}
}
if _, err := DecodeAsk(hold, []byte(`{"v":2,"ask":{}}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("DecodeAsk of another version = %v", err)
}
}
@@ -1,23 +0,0 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
)
require (
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
)
@@ -1,100 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askAgainTool asks one of the assigner's asks again by hand (issue 1966, ADR 0668 §5): the recovery of an ask that ended at
// the operator channel without an answer when the event that would have asked it again was not heard.
const askAgainTool = "assigner_ask_again"
// asksState and answersState are the assigner's asks and the teller's answers, as the runtime names them.
const (
asksState = "asks"
answersState = buckets.TellerModule + ".answers"
)
// askAgainByHand writes the ask at key again under a new asking, by the rules the teller's ask-ended writes it by (assign.AskedAgain:
// a proposal's authorising ask and a put's record ask are not), so the teller asks it again; an asking the operator answered
// is refused, naming the answer. The act is recorded through the seat mesh-hand-acts.
func askAgainByHand(st store, ask func(string, any) (json.RawMessage, error), key, why, by string, now time.Time) (any, error) {
switch {
case key == "":
return nil, errors.New("name the ask's key, as the tool asks lists it")
case why == "":
return nil, errors.New("a hand ask again says why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("a hand ask again names who asks it, and the runtime proved no caller")
}
current, _, found, err := st.get(asksState, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the assigner holds no ask under %s: the tool asks lists them", key)
}
_, routerAsk, err := assign.DecodeAsk(key, current)
if err != nil {
return nil, err
}
if raw, _, found, err := st.get(answersState, key); err == nil && found {
if a, err := meshdeliveryclient.DecodeAnswer(raw); err == nil && strings.HasPrefix(a.Warrant, routerAsk.ID+"@") {
return nil, fmt.Errorf("the asking %s was answered %q (%s): it is not asked again", routerAsk.ID, a.Option, a.Words)
}
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRecord {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
if raw, _, found, err := st.get(putsState, pkey); err == nil && found {
if p, err := puts.Decode(raw); err == nil {
put = &p
}
}
}
}
value, refused := assign.AskedAgain(tellerevents.Ended{Key: key, Ask: routerAsk.ID, Outcome: occ.OutcomeCancelled}, current, put, now)
if refused != "" {
return nil, errors.New(refused)
}
if err := st.put(asksState, key, value); err != nil {
return nil, fmt.Errorf("the ask %s is not written again: %w", key, err)
}
_, next, _ := assign.DecodeAsk(key, value)
taken, rerr := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askAgainTool + " " + key, Why: why + " (" + by + ")", Cause: "ask-again"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
}
return map[string]any{"key": key, "ended": routerAsk.ID, "asked": next.ID, "by": by, "hand-act": recorded}, nil
}
// askAgainTools is the tool over the runtime's states.
func askAgainTools(st store, ask func(string, any) (json.RawMessage, error), now func() time.Time) []stdio.Tool {
return []stdio.Tool{{Name: askAgainTool, Description: "Ask one of the assigner's asks of the operator again (issue 1966): the ask " +
"under key is written again under a new asking, which the teller asks on the operator channel, as when the channel says it " +
"ended without an answer. Refused for an asking the operator answered, a proposal's authorising ask (the seat's verb again " +
"asks it) and a record ask a put asked. Recorded as a hand act.",
Input: schema(map[string]string{"key": "the ask's key, as the tool asks lists it", "why": "why it is asked again"},
[]string{"key", "why"}),
Run: func(a map[string]any) (any, error) {
key, _ := a["key"].(string)
why, _ := a["why"].(string)
return askAgainByHand(st, ask, key, why, stdio.CallerOf(a), now())
}}}
}
@@ -1,239 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/process"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// askEndedSubject is the teller's event ask-ended, as the manifest's consumes names it (`mesh-delivery-teller.ask-ended`).
var askEndedSubject = process.Account{Module: buckets.TellerModule}.EventSubject(tellerevents.AskEnded)
// staleOf says why the ask at a key is stale, as the pass reads it (passer.staleAsk), or "" when it is not or when the
// trunk is not read; nil reads nothing.
type staleOf func(key string) string
// staleOfWorld is staleOf over the world the pass reads: the same rule as the pass, over the same cache.
func staleOfWorld(w *world) staleOf {
p := &passer{w: w}
return func(key string) string {
trunk := w.trunk()
if len(trunk) == 0 {
return ""
}
return p.staleAsk(key, trunk)
}
}
// hearAskEnded hears the teller's ask-ended on the assigner's one hub of its durable consumer on EVENTS (registered before
// the hub starts): an ask of the assigner's that ended at the operator channel without an answer while its key stands is
// written again under a new asking, so the teller asks it again (issue 1966, note 151). asksKV is the assigner's asks; the
// puts are opened at each event, since a put's record ask is not written again (ADR 0546).
func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, logf func(string, ...any)) error {
return hub.Handle(askEndedSubject, func(m jetstream.Msg) {
var putsKV jetstream.KeyValue
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
if err := askAgain(ctx, asksKV, putsKV, stale, m.Data(), time.Now(), logf); err != nil {
logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
})
}
// channelAsksVerb is the operator-channel seat's verb asks, read at the assigner's consumer's bind (a seat verb: no module
// named).
const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks
// readEndedAtBind registers on the hub, before it starts, one read of the operator-channel seat's asks at each bind of the
// assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155,
// ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it
// (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start.
func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, logf func(string, ...any)) {
if asksKV == nil || bus == nil {
return
}
hub.OnBound(func(created time.Time) {
go func() {
if err := endedAtBind(ctx, js, asksKV, stale, bus, wait, created, time.Now(), logf); err != nil {
logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err)
}
}()
})
}
// callWait is how a kept call the assigner waits on is said (novox/hq ADR 0668 §3): heard says the callee's machine is
// heard, now is the clock, and findings the assigner's findings, where the call unanswered past
// meshconditionsclient.CallUnansweredAfter is the finding call-unanswered while heard, cleared when it is answered.
type callWait struct {
heard func() bool
now func() time.Time
findings assign.Findings
}
// asksCallID is the finding's id for the bind's read of the operator channel's asks.
const asksCallID = "operator-channel-asks"
// keep makes call again after every transport fault (busretry: a request that reached no one, or one whose deadline
// passed) until it is answered or ctx ends, each try itself a kept call; past the bound with the callee heard it writes
// the finding call-unanswered, and clears it when answered. No timer: each try is the wait.
func (cw callWait) keep(ctx context.Context, id, callee string, call func() (json.RawMessage, error), logf func(string, ...any)) (json.RawMessage, error) {
clock := cw.now
if clock == nil {
clock = time.Now
}
since, said := clock(), false
key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered)
for {
answer, err := call()
if err == nil || !(busretry.Undelivered(err) || busretry.TimedOut(err)) || ctx.Err() != nil {
if said && cw.findings.Delete != nil {
if derr := cw.findings.Delete(key); derr != nil {
logf("the finding %s is not cleared: %v", key, derr)
}
}
return answer, err
}
if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil {
continue
}
eerr := cw.findings.Put(assign.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since).Finding(clock()))
if eerr != nil {
logf("the finding %s is not written: %v", key, eerr)
continue
}
said = true
logf("%s is unanswered since %s and its callee's machine is heard: the finding %s", callee, since.UTC().Format(time.RFC3339), key)
}
}
// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and
// each ended asking written again.
func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
keys, err := asksKV.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil
}
if err != nil {
return err
}
held := map[string][]byte{}
for _, key := range keys {
if e, err := asksKV.Get(ctx, key); err == nil {
held[key] = e.Value()
}
}
query, ok := assign.ChannelAsksQuery(held)
if !ok {
return nil
}
answer, err := wait.keep(ctx, asksCallID, channelAsksVerb, func() (json.RawMessage, error) { return bus.CallKept(channelAsksVerb, query) }, logf)
if err != nil {
return err
}
ended, cut, err := assign.EndedAtChannel(held, answer)
if err != nil {
return err
}
if cut != "" {
logf("%s at the bind: %s", channelAsksVerb, cut)
}
var putsKV jetstream.KeyValue
if js != nil {
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
putsKV = kv
}
}
for _, e := range ended {
body, err := tellerevents.EncodeEnded(e)
if err == nil {
err = askAgain(ctx, asksKV, putsKV, stale, body, now, logf)
}
if err != nil {
return err
}
}
logf("%s at the bind of %s: %d of %d held asks ended at the channel without an answer", channelAsksVerb, created.Format(time.RFC3339), len(ended), len(held))
return nil
}
// askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the
// asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that
// could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped.
func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, stale staleOf, data []byte, now time.Time, logf func(string, ...any)) error {
e, err := tellerevents.DecodeEnded(data)
if err != nil {
logf("the teller's %s holds what is not an ended ask, and is dropped: %v", tellerevents.AskEnded, err)
return nil
}
if asksKV == nil {
return errors.New("the assigner's asks are not open")
}
var current []byte
var rev uint64
entry, err := asksKV.Get(ctx, e.Key)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
current, rev = entry.Value(), entry.Revision()
}
var put *puts.Put
if k, err := asks.ParseAnyAskKey(e.Key); err == nil && k.Kind == asks.AskKindRecord && putsKV != nil {
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
pe, err := putsKV.Get(ctx, pkey)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
if p, err := puts.Decode(pe.Value()); err == nil {
put = &p
}
}
}
}
value, why := assign.AskedAgain(e, current, put, now)
if why == "" && stale != nil {
// An ask the pass takes away is not written again: it is about a move or a composition that has been left behind, and the
// pass removes it (issue 1966 note 165).
if s := stale(e.Key); s != "" {
why = "it is stale and left to the pass to take away: " + s
}
}
if why != "" {
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is not written again: %s", e.Key, e.Ask, e.Outcome, why)
return nil
}
if _, err := asksKV.Update(ctx, e.Key, value, rev); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
logf("%s: the ask %s ended (%s) without an answer, and the key changed since: it is not written again", e.Key, e.Ask, e.Outcome)
return nil
}
return err
}
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is written again under a new asking, which the teller asks", e.Key, e.Ask, e.Outcome)
return nil
}
@@ -1,117 +0,0 @@
package main
import (
"errors"
"io/fs"
"os"
"slices"
"strings"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"mesh-delivery-assigner/internal/judge"
)
// Which proposals the assigner asks of the operator is its setting ask-operator-for (design 47 G A-425, A-425a): a list of change
// kinds, set at mesh level with mesh-config's config, its default take-off, remove-node and stops. A proposal none of whose changes
// is listed is applied at once when its verdict passes, said in the log and by assignments-changed; one with a listed kind is
// asked as before. The setting is read from the file the manifest declares with the content ${config:ask-operator-for}, which the
// environment names (A-112), at every pass, and a change of it is said.
// KindStops is the change kind of a verdict whose answers say a node would stop running a module.
const KindStops = "stops"
// defaultAskOperatorFor is the setting's default.
var defaultAskOperatorFor = []string{mmac.OpTakeOff, mmac.OpRemoveNode, KindStops}
// askKinds are the kinds the setting may name.
var askKinds = []string{mmac.OpPlace, mmac.OpTakeOff, mmac.OpPin, mmac.OpUnpin, mmac.OpHold, mmac.OpUnhold, mmac.OpDecide, mmac.OpConfig,
mmac.OpAddNode, mmac.OpRemoveNode, KindStops}
// parseAskOperatorFor reads the setting's value: kinds separated by commas or spaces; empty is the default. A kind it does not
// know is answered apart and named nowhere in the list.
func parseAskOperatorFor(raw string) (kinds, unknown []string) {
fields := strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
if len(fields) == 0 {
return slices.Clone(defaultAskOperatorFor), nil
}
for _, f := range fields {
f = strings.ToLower(strings.TrimSpace(f))
switch {
case !slices.Contains(askKinds, f):
unknown = append(unknown, f)
case !slices.Contains(kinds, f):
kinds = append(kinds, f)
}
}
slices.Sort(kinds)
return kinds, unknown
}
// readAskOperatorFor reads the setting's file: the default for no path or no file.
func readAskOperatorFor(path string) (kinds, unknown []string, err error) {
if path == "" {
k, u := parseAskOperatorFor("")
return k, u, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
k, u := parseAskOperatorFor("")
return k, u, nil
}
if err != nil {
return nil, nil, err
}
k, u := parseAskOperatorFor(string(raw))
return k, u, nil
}
// askOperatorFor is the setting as this pass read it: the file read again each pass, a change said once, a file that cannot be
// read keeping what was read last (the default before any).
func (p *passer) askOperatorFor() []string {
kinds, unknown, err := readAskOperatorFor(p.askForFile)
if err != nil {
if p.askFor == nil {
p.askFor = slices.Clone(defaultAskOperatorFor)
}
p.logf("the setting ask-operator-for cannot be read from %s, so %s stands: %v", p.askForFile, strings.Join(p.askFor, ", "), err)
return p.askFor
}
if !slices.Equal(kinds, p.askFor) {
words := "the setting ask-operator-for is " + strings.Join(kinds, ", ") + ": a proposal with a change of another kind only is applied with no ask"
if len(unknown) > 0 {
words += "; kinds it names that are none are left out: " + strings.Join(unknown, ", ")
}
p.logf("%s", words)
p.askFor = kinds
}
return p.askFor
}
// asksFor says whether a proposal of these change kinds is asked of the operator.
func (p *passer) asksFor(kinds []string) bool {
set := p.askOperatorFor()
for _, k := range kinds {
if slices.Contains(set, k) {
return true
}
}
return false
}
// proposalKinds are the change kinds of a proposal whose verdict is v: each change's op, and stops when an answer says a node
// would stop running a module.
func proposalKinds(prop mma.Proposal, v mma.Verdict) []string {
var out []string
for _, c := range prop.Changes {
if c.Op != "" && !slices.Contains(out, c.Op) {
out = append(out, c.Op)
}
}
if len(judge.Stops(v)) > 0 {
out = append(out, KindStops)
}
slices.Sort(out)
return out
}
@@ -0,0 +1,161 @@
package main
import (
"context"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-assigner/internal/assign"
)
// AssignerModule is the asker the assigner's compose asks name.
const AssignerModule = "mesh-delivery-assigner"
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
// second ask builds nothing.
type composeAsks struct {
ctx context.Context
asker buildrun.Asker
wake func()
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
asks map[string]*composeAsk
}
type composeAsk struct {
id string
cancel context.CancelFunc
ran *assign.Ran
}
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
}
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
// A job its layout refuses ends at once, failed with why: never silent.
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
a.mu.Lock()
defer a.mu.Unlock()
if k, ok := a.asks[key]; ok {
return k.ran, k.id
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
At: a.now(), Compose: &job}
ctx, cancel := context.WithCancel(a.ctx)
k := &composeAsk{id: ask.ID, cancel: cancel}
a.asks[key] = k
if err := a.asker.Put(ctx, ask); err != nil {
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
cancel()
return k.ran, k.id
}
go func() {
step, err := a.asker.Wait(ctx, ask)
if ctx.Err() != nil {
return
}
ran := assign.RanOf(step)
if err != nil {
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
}
a.mu.Lock()
k.ran = &ran
a.mu.Unlock()
a.wake()
}()
return nil, k.id
}
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
func (a *composeAsks) sweep(seen map[string]bool) {
a.mu.Lock()
defer a.mu.Unlock()
for key, k := range a.asks {
if !seen[key] {
k.cancel()
delete(a.asks, key)
}
}
}
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
// asks. The keys it used are marked in seen.
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
waiting := func(words string) delivery.CompositionVerdict {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
}
head, err := assign.HeadJob(m, ch)
if err != nil {
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
}
var pending []string
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
seen[key] = true
headRan, id := p.asks.get(key, head)
if headRan == nil {
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
}
var readers []assign.Reader
var notJudged []string
for _, module := range sortedNames(assign.ReaderModules) {
repository := assign.ReaderModules[module]
var commits []string
member := false
if h := ch.Members[repository]; h != "" {
commits, member = []string{h}, true
} else {
commits = p.w.runningCommits(module)
}
if len(commits) == 0 {
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+
"trunk build of it, and the group holds no member of "+repository)
continue
}
for _, commit := range commits {
job, err := assign.ReaderJob(ch, module, repository, commit)
if err != nil {
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
continue
}
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
seen[rkey] = true
ran, rid := p.asks.get(rkey, job)
if ran == nil {
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
continue
}
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
}
}
if len(pending) > 0 {
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
}
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
v.Warnings = append(v.Warnings, notJudged...)
return v
}
func sortedNames(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -1,162 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks
// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps
// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first.
const (
asksTool = "assigner_asks"
askDropTool = "assigner_ask_drop"
)
// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total).
const asksAnswerMost = 768 << 10
// takenReader is what assigner_asks reads of the state taken (stdio.KeptState).
type takenReader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.<ask>) and the drops of
// asks not pulled yet (dropped.<ask>), each list cut at the answer's bound with capped and total.
func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) {
w, err := waiting(meshwants.Seat)
if err != nil {
return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err)
}
keys, err := taken.Keys()
if err != nil {
return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err)
}
slices.Sort(keys)
// An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a
// nak (H-A-410's addenda): the bus says what it knows, the state taken the rest.
size := 0
waits := make([]map[string]any, 0, len(w.Asks))
for _, a := range w.Asks {
size += len(a.ID) + len(a.Module) + 64
state := "waiting"
if a.Again {
state = "waiting again"
if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) {
state = "held"
}
}
waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state})
}
takenIDs, dropped := []string{}, []map[string]any{}
total, capped := 0, w.Capped
for _, k := range keys {
if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok {
total++
if size += len(id) + 4; size > asksAnswerMost {
capped = true
continue
}
takenIDs = append(takenIDs, id)
continue
}
id, ok := strings.CutPrefix(k, droppedPrefix)
if !ok {
continue
}
total++
e, err := taken.Get(k)
if err != nil || e == nil {
continue
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
continue
}
if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost {
capped = true
continue
}
dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At})
}
return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil
}
// asksTools are the module's tools of the seat's work queue.
func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{
{Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " +
"that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " +
"cut at 768 KB with capped and total. Read-only.",
Input: schema(map[string]string{}, nil),
Run: func(map[string]any) (any, error) {
return asksOf(waiting, taken)
}},
{Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " +
"first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " +
"its step could not run naming the drop and who asked it.",
Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}),
Run: func(args map[string]any) (any, error) {
id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why"))
if id == "" || why == "" {
return nil, errors.New("name the ask and why it is ended")
}
by := stdio.CallerOf(args)
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why})
if err != nil {
return nil, err
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err)
}
out["hand-act"] = taken.Log
return out, nil
}},
}
}
// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat
// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process.
const keepNowTool = "assigner_keep_now"
func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " +
"hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " +
"the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).",
Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}),
Run: func(args map[string]any) (any, error) {
why := strings.TrimSpace(arg(args, "why"))
if why == "" {
return nil, errors.New("say why the keeping runs now")
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err)
}
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why})
if err != nil {
return nil, err
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err)
}
out["hand-act"] = taken.Log
return out, nil
}}}
}
@@ -1,74 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §717: the assigner's asks are the seat mesh-wants' state asks, so every key its asks package writes is one of
// that state's key forms, read back to the same parts, and each value is the seat's Ask.
func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
const hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
const fp, digest = "src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
must := func(k string, err error) string {
t.Helper()
if err != nil {
t.Fatal(err)
}
return k
}
three := func(parse func(string) (string, string, string, bool)) func(string) []string {
return func(k string) []string {
if a, b, c, ok := parse(k); ok {
return []string{a, b, c}
}
return nil
}
}
for _, row := range []struct {
key string
parse func(string) []string
want []string
}{
{must(asks.HoldAskKey("web", fp)), func(k string) []string {
if a, b, ok := meshwants.ParseAsksByHoldModuleFingerprintKey(k); ok {
return []string{a, b}
}
return nil
}, []string{"web", hex}},
{must(asks.RestoreAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRestoreNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.BackupAskKey("ace", "web", fp)), three(meshwants.ParseAsksByBackupNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.OutsideAskKey("ace", "web", fp)), three(meshwants.ParseAsksByOutsideNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RevertAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRevertNodeModuleFingerprintKey), []string{"ace", "web", hex}},
{must(asks.RecordAskKey("ace", "web", digest)), three(meshwants.ParseAsksByRecordNodeModuleDigestKey), []string{"ace", "web", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}},
{must(asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")), func(k string) []string {
if a, ok := meshwants.ParseAsksByAssignmentsProposalKey(k); ok {
return []string{a}
}
return nil
}, []string{"p-20261028t183000-1a2b3c"}},
} {
got := row.parse(row.key)
if len(got) != len(row.want) {
t.Errorf("%s is no key form of mesh-wants' asks: read as %v", row.key, got)
continue
}
for i := range got {
if got[i] != row.want[i] {
t.Errorf("%s reads as %v, want %v", row.key, got, row.want)
break
}
}
}
key := must(asks.HoldAskKey("web", fp))
raw, err := asks.EncodeAsk(key, asks.Ask{Ask: json.RawMessage(`{"id":"x"}`), Asking: 2})
if err != nil {
t.Fatal(err)
}
if v, _, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
}
}
@@ -1,221 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the
// builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's
// `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard.
// builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded).
type builtEnds struct {
mu sync.Mutex
ends map[string]builds.BuildEnded
}
func (b *builtEnds) key(module, fp string) string { return module + "@" + fp }
// put takes one build.ended in.
func (b *builtEnds) put(data []byte) error {
var e builds.BuildEnded
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a build.ended that is no end: %w", err)
}
if e.Module == "" || e.Fingerprint == "" {
return fmt.Errorf("a build's end naming no module or fingerprint: %s", data)
}
if e.Outcome == "" { // an end said before A-162 is a done build's
e.Outcome = builds.BuildDone
}
b.mu.Lock()
defer b.mu.Unlock()
if b.ends == nil {
b.ends = map[string]builds.BuildEnded{}
}
b.ends[b.key(e.Module, e.Fingerprint)] = e
return nil
}
// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended.
func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) {
b.mu.Lock()
defer b.mu.Unlock()
e, ok := b.ends[b.key(module, fp)]
return e, ok
}
// partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `<module>:<artifact>`), with the
// node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose
// parts the version's release says (releasedVersions).
func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) {
b.mu.Lock()
e, ok := b.ends[b.key(module, fp)]
b.mu.Unlock()
if !ok || (e.KeptOn == "" && e.Branch == "") {
return assign.KeptBuild{}, false
}
parts := map[string]string{}
for name, digest := range e.Made {
parts[module+":"+name] = digest
}
return assign.KeptBuild{Parts: parts, KeptOn: e.KeptOn}, true
}
// releasedVersions are the catalog's version-released heard, by module and source revision.
type releasedVersions struct {
mu sync.Mutex
seen map[string]assign.KeptBuild
// newest is, by module, the last version-released heard of it (delivered.go).
newest map[string]releaseHeard
}
func (r *releasedVersions) key(module string, source int64) string {
return fmt.Sprintf("%s@%d", module, source)
}
// put takes one version-released in.
func (r *releasedVersions) put(data []byte) error {
var e meshcatalog.VersionReleasedEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-released that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-released naming no module or version: %s", data)
}
parts := map[string]string{}
for _, a := range e.Artifacts {
parts[partRef(e.Module, a.Artifact)] = a.Digest
}
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]assign.KeptBuild{}
}
r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts}
if r.newest == nil {
r.newest = map[string]releaseHeard{}
}
h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source}
if e.At != nil {
h.At = *e.At
}
r.newest[e.Module] = h // the stream's order is the order of release
return nil
}
// withParts is the version of module whose release named exactly parts (`<module>:<artifact>` -> digest), the newest such when
// more than one did; false when none did or parts is empty (design 47 G A-421).
func (r *releasedVersions) withParts(module string, parts map[string]string) (assign.KeptBuild, bool) {
if len(parts) == 0 {
return assign.KeptBuild{}, false
}
r.mu.Lock()
defer r.mu.Unlock()
var out assign.KeptBuild
found := false
for key, k := range r.seen {
if m, _, ok := strings.Cut(key, "@"); !ok || m != module || !maps.Equal(k.Parts, parts) {
continue
}
if !found || compareVersion(k.Version, out.Version) > 0 {
out, found = k, true
}
}
return out, found
}
// madeOf is what module's done build at fp made, by `<module>:<artifact>`, as its build.ended names it; none for a build not heard
// or not done (design 47 G A-421).
func (b *builtEnds) madeOf(module, fp string) map[string]string {
e, ok := b.endOf(module, fp)
if !ok || e.Outcome != builds.BuildDone {
return nil
}
parts := map[string]string{}
for name, digest := range e.Made {
parts[module+":"+name] = digest
}
return parts
}
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
r.mu.Lock()
defer r.mu.Unlock()
k, ok := r.seen[r.key(module, source)]
return k, ok
}
// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and
// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703).
var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream)
// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was
// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before
// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner
// starts again.
func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) {
if err := ends.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf, process.FromFirst()) // read whole from its first message whether the bus keeps it last per subject or by age (A-181, A-233)
if err != nil {
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
return
}
nh.Heard(buildsStreamName)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// compareVersion orders two MAJOR.MINOR.PATCH versions by their numbers; a part that is no number counts as 0.
func compareVersion(a, b string) int {
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
for i := 0; i < 3; i++ {
var x, y int
if i < len(pa) {
x, _ = strconv.Atoi(pa[i])
}
if i < len(pb) {
y, _ = strconv.Atoi(pb[i])
}
if x != y {
return x - y
}
}
return 0
}
// partRef is a release's artifact as a part's key, `<module>:<artifact>`: the contract's VersionArtifact names the artifact
// reference whole (an ArtifactRef), as the catalog's version-released says it; a bare name is taken as the module's (A-96).
func partRef(module, artifact string) string {
if strings.Contains(artifact, ":") {
return artifact
}
return module + ":" + artifact
}
@@ -1,112 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
)
// Design 47 G §379, §526, A-96: a build.ended heard names a kept build's or another branch's parts by digest (`<module>:<artifact>`) and its
// keeping node; a build of the trunk not kept names none, its parts being the release's.
func TestABuildEndedHeardNamesAKeptBuildsPartsAndItsNode(t *testing.T) {
put := func(b *builtEnds, e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := b.put(raw); err != nil {
t.Fatal(err)
}
}
made := map[string]string{"program": "sha256:1"}
var b builtEnds
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "a", Made: made})
if _, ok := b.partsOf("node-engine", "a"); ok {
t.Error("a trunk build not kept names parts")
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "b", Made: made, KeptOn: "ace"})
if k, ok := b.partsOf("node-engine", "b"); !ok || k.KeptOn != "ace" || !reflect.DeepEqual(k.Parts, map[string]string{"node-engine:program": "sha256:1"}) {
t.Errorf("kept build = %+v, %v", k, ok)
}
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "c", Made: made, Branch: "cand"})
if k, ok := b.partsOf("node-engine", "c"); !ok || k.KeptOn != "" || len(k.Parts) != 1 {
t.Errorf("branch build = %+v, %v", k, ok)
}
if err := b.put([]byte(`{"module":"x"}`)); err == nil {
t.Error("an end naming no fingerprint is kept")
}
}
// Design 47 G §379, A-96: a version-released heard names the version and the parts' digests of the module's source revision.
func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
var r releasedVersions
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
if err := r.put(raw); err != nil {
t.Fatal(err)
}
want := assign.KeptBuild{Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}}
if k, ok := r.of("mail", 7); !ok || !reflect.DeepEqual(k, want) {
t.Errorf("released = %+v, %v", k, ok)
}
if _, ok := r.of("mail", 8); ok {
t.Error("another source revision names a version")
}
}
// Design 47 G §379, A-96, A-421; H §764, H-A-48: the trunk's entry, which the catalog stamps with no source, takes its version and
// parts from the version-released whose parts are what its build.ended made; a kept build from its end; no revision kept, another
// fingerprint and a build whose parts no release names name none, and an unheard release is said once.
func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsBuildsParts(t *testing.T) {
w := newWorld()
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
if err := w.released.put(raw); err != nil {
t.Fatal(err)
}
endRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "k", Made: map[string]string{"image": "sha256:2"}, KeptOn: "g14"})
if err := w.ends.put(endRaw); err != nil {
t.Fatal(err)
}
var logs []string
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}, logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
named := p.namedBuild("mail", "t")
if _, ok := named("t"); ok {
t.Error("a trunk entry whose build.ended is not heard names a version")
}
_, _ = named("t")
if k, ok := named("k"); !ok || k.KeptOn != "g14" || k.Version != "" {
t.Errorf("kept build = %+v, %v", k, ok)
}
if _, ok := named("other"); ok {
t.Error("another fingerprint names a release")
}
// A build held back at an older fingerprint names the release whose parts are exactly what it made.
heldRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "old", Made: map[string]string{"image": "sha256:9"}})
if err := w.ends.put(heldRaw); err != nil {
t.Fatal(err)
}
if k, ok := named("old"); !ok || k.Version != "1.2.3" {
t.Errorf("a held build whose parts were released = %+v, %v; want 1.2.3", k, ok)
}
// The entry's module value carries no source stamp: the catalog writes none (H-A-48).
if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{}}); v.Fp2 != "t" {
t.Errorf("the entry's module value is %+v", v)
}
made, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "t", Made: map[string]string{"image": "sha256:9"}})
if err := w.ends.put(made); err != nil {
t.Fatal(err)
}
p.graph.source = 8 // a later merge that did not rebuild mail
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" {
t.Errorf("the trunk entry whose build made the released digests: %+v, %v", k, ok)
}
if len(logs) != 1 || !strings.Contains(logs[0], "the release of mail at fingerprint t is not heard: no version-released of it names the parts its build.ended made") {
t.Errorf("logs = %q, want the unheard release said once, naming the module and the fingerprint", logs)
}
p.graph.ok = false
if _, ok := p.namedBuild("mail", "t")("t"); ok {
t.Error("no revision kept names a release")
}
}
@@ -1,24 +0,0 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G
// A-181: what was built is heard, never read from the builder's bucket).
func endDone(t testing.TB, w *world, module, fp, manifest string) {
t.Helper()
data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest})
if err != nil {
t.Fatal(err)
}
if err := w.ends.put(data); err != nil {
t.Fatal(err)
}
}
// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507).
const trunkBranch = "main"
@@ -1,72 +0,0 @@
package main
import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// addedOf is the `added` of node's candidate (design 47 G A-316, §379, §421): by module, each module the candidate adds to node
// (laid over current) with no want there, the version and parts the want it would write carries, taken from the same code that
// writes a want's (namedBuild, the want's build at the fingerprint of the branch the want's code takes). A module whose build names none is not listed.
func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]mma.Assignments) map[string]mma.AddedModule {
if laid == nil {
return nil
}
placed := map[string]string{}
for _, m := range laid.Modules {
placed[m.Module] = m.Branch
}
var out map[string]mma.AddedModule
for _, module := range plc.ChangeOf(current[node], *laid).Added {
if key, err := mwc.WantKey(node, module); err != nil || p.w.has(buckets.Wants(), key) {
continue
}
// The branch the want's code takes: the one the candidate's placement names, else the module's default (groups, wants).
branch := p.w.trunkName()
if p.ends.following() {
branch = p.ends.branchOf(module, placed[module])
}
entry := p.w.trunkOf(branch)[module]
fp := entry.Fp2
if fp == "" {
p.skippedAdd(node, module, "", "the branch "+branch+" names no fingerprint of it")
continue
}
k, ok := p.namedBuild(module, fp)(fp)
if !ok {
p.skippedAdd(node, module, fp, "no version-released of it is heard for that fingerprint, so the candidate carries no version and no parts")
continue
}
if out == nil {
out = map[string]mma.AddedModule{}
}
added := mma.AddedModule{Version: k.Version, Parts: k.Parts}
if k.Version != "" {
added.Parts = nil // a released build is named by its version alone (design 47 G A-430, A-430a)
}
out[module] = added
}
return out
}
// candidateOf is the candidate of proposal id for node: the assignments laid over it, and what it adds (addedOf).
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
}
// skippedAdd says once per node, module and fingerprint why a module the candidate adds is not listed in `added`.
func (p *passer) skippedAdd(node, module, fp, why string) {
key := node + "/" + module + "@" + fp + ": " + why
if p.skipped[key] {
return
}
if p.skipped == nil {
p.skipped = map[string]bool{}
}
p.skipped[key] = true
if p.logf != nil {
p.logf("the candidate adds %s to %s with no parts: %s (fingerprint %q)", module, node, why, fp)
}
}
@@ -1,63 +0,0 @@
package main
import (
"encoding/json"
"reflect"
"testing"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G A-316: the candidate of a node carries, for each module it adds there without a want, the version and parts the
// want it would write carries; a module with a want there, or a build that names none, is not listed.
func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
w := newWorld()
for _, m := range []string{"mail", "web", "db"} {
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
}
w.first = "main"
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
if err := w.released.put(rel); err != nil {
t.Fatal(err)
}
built, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-mail", Made: map[string]string{"image": "sha256:9"}})
if err := w.ends.put(built); err != nil {
t.Fatal(err)
}
end, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: "fp-db", Made: map[string]string{"program": "sha256:3"}, KeptOn: "g14"})
if err := w.ends.put(end); err != nil {
t.Fatal(err)
}
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 9}} // a later merge: the release is matched by the parts its build made (A-421, H-A-48)
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
c := p.candidateOf("p1", "ace", laid, current)
want := map[string]mma.AddedModule{
"mail": {Version: "1.2.3"}, // a released build is named by its version alone (A-430a)
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
}
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
t.Errorf("candidate = %+v, added want %+v", c, want)
}
if got := p.candidateOf("p1", "ace", nil, current); got.Added != nil {
t.Errorf("a candidate taking the key away adds %+v", got.Added)
}
// A placement on a watched branch is added at that branch's fingerprint, as the want's code takes it, not the trunk's.
w.putModule("feat", "mail", meshcatalog.ModulesByModuleValue{Fp2: "fp-feat"}, graph.Node{}, nil)
feat, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-feat", Made: map[string]string{"program": "sha256:4"}, KeptOn: "g14"})
if err := w.ends.put(feat); err != nil {
t.Fatal(err)
}
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = w.trunkName
p.ends.on.Store(true)
onFeat := &mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "mail", Branch: "feat"}}}
if got, want := p.candidateOf("p2", "g14", onFeat, nil).Added, map[string]mma.AddedModule{"mail": {Parts: map[string]string{"mail:program": "sha256:4"}}}; !reflect.DeepEqual(got, want) {
t.Errorf("a placement on branch feat adds %+v, want %+v", got, want)
}
}
@@ -1,113 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"strings"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/assigngraph"
)
// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-<id>`'s catalog graph revision,
// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is
// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate.<group>.<heads>;
// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing.
// runGroupPrefix is a run on request's candidate group before its id, `run-<id>` (A-160).
const runGroupPrefix = "run-"
// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the
// candidate's group and heads and the revision its event names.
func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
after := ""
for {
a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after})
if err != nil {
return half, err
}
if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group {
return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision)
}
half.Revision = *a.Revision
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
after = a.Next
}
}
// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a
// group's) is passed over.
func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error {
if !strings.HasPrefix(ev.Group, runGroupPrefix) {
return nil
}
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return errors.New("the assigner keeps no assignment graph, so no candidate's is kept")
}
half, err := readCandidateHalf(call, ev)
if err != nil {
return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err)
}
in := p.graphInputs()
in.Halves = maps.Clone(in.Halves)
in.Halves[half.Revision.Branch] = half
c := assigngraph.Compute(in)
for _, u := range c.Uncomputed {
p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u)
}
rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock())
if err != nil {
return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err)
}
if fresh {
p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision)
}
return nil
}
// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235).
func (p *passer) dropCandidate(ctx context.Context, group string) error {
g := p.graph
if g == nil || g.objects == nil {
return nil
}
o, ok := g.objects.(assigngraph.Remover)
if !ok {
return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group)
}
n, err := assigngraph.DropCandidate(ctx, o, group)
if err == nil && n > 0 {
p.logf("the candidate %s's assignment graph is dropped: its run ended", group)
}
return err
}
// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it.
func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
raw, err := call(mcc.RevisionVerb, in)
if err != nil {
return meshcatalog.RevisionAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
return a, nil
}
@@ -0,0 +1,180 @@
//go:build integration
package main
import (
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
type holder struct {
mu sync.Mutex
asks []delivery.StepAsk
}
func (h *holder) taken() []delivery.StepAsk {
h.mu.Lock()
defer h.mu.Unlock()
return append([]delivery.StepAsk(nil), h.asks...)
}
func (b *bed) hold(t *testing.T) *holder {
t.Helper()
seat, _ := delivery.AskStream(delivery.AskGo)
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
h := &holder{}
cc, err := cons.Consume(func(m jetstream.Msg) {
a, err := delivery.DecodeStepAsk(m.Data())
if err != nil || a.Compose == nil {
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
m.Ack()
return
}
h.mu.Lock()
h.asks = append(h.asks, a)
h.mu.Unlock()
req, err := composer.Decode(a.Compose.Request)
if err == nil {
err = composer.Fill(&req, func(digest string) ([]byte, error) {
key, _ := delivery.ManifestsKey(digest)
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
if err != nil {
return nil, err
}
man, err := delivery.DecodeManifest(e.Value())
return []byte(man.JSON), err
})
}
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else if v, err := composer.Run(req); err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
} else {
vr, _ := json.Marshal(v)
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
}
raw, err := delivery.EncodeStep(step)
if err != nil {
t.Errorf("the step cannot be written: %v", err)
}
key, _ := delivery.AskStepKey("ace", a.ID)
if _, err := steps.Put(b.ctx, key, raw); err != nil {
t.Errorf("the step %s: %v", key, err)
}
m.Ack()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(cc.Stop)
return h
}
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
// head's composer was built with, and asks nothing more on a later pass of the same heads.
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
running := strings.Repeat("3", 40)
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
b.built(t, "mesh-controller", "novox/mesh-controller", running)
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey("feat/x")
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
b.start(t)
key, _ := delivery.CompositionKey("feat/x", hash)
// No holder takes the asks yet: the composition waits, naming them.
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
}
h := b.hold(t)
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
t.Fatalf("composition = %+v, want a pass on g14", v)
}
asks := h.taken()
if len(asks) != 2 {
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
}
byRepo := map[string]delivery.StepAsk{}
for _, a := range asks {
byRepo[a.Compose.Fetch["repository"].(string)] = a
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
t.Errorf("ask = %+v", a)
}
}
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
t.Errorf("the head's ask = %+v", a.Compose)
}
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
t.Errorf("the running controller's ask = %+v", a.Compose)
}
// A later pass of the same heads (woken by any change) asks nothing more.
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
time.Sleep(500 * time.Millisecond)
if n := len(h.taken()); n != 2 {
t.Errorf("a later pass of the same heads asked again: %d asks", n)
}
}
// verdict waits for the composition under key to satisfy ok, and answers it.
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
t.Helper()
var last delivery.CompositionVerdict
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
t.Fatal(err)
}
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
t.Fatal(err)
}
if ok(last) {
return last
}
}
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
return last
}
@@ -1,78 +0,0 @@
package main
import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
// it was refused or cannot be read.
type publishedRow struct {
Key string `json:"key"`
Module string `json:"module"`
FP string `json:"fp,omitempty"`
Digest string `json:"digest,omitempty"`
Refused string `json:"refused,omitempty"`
Unread string `json:"unread,omitempty"`
}
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
// names its node first): each value decoded and digested here, as the node-engine digests it (nodeengineclient.ComposedDigest).
// A value or key that does not read is named with why, never left out.
func published(values map[string][]byte) map[string][]publishedRow {
byNode := map[string]map[string][]byte{}
for key, raw := range values {
node, _, _ := strings.Cut(key, ".")
if byNode[node] == nil {
byNode[node] = map[string][]byte{}
}
byNode[node][key] = raw
}
out := map[string][]publishedRow{}
for node, keys := range byNode {
var rows []publishedRow
for key, raw := range keys {
row := publishedRow{Key: key}
_, module, part, isNode, err := nodeengineclient.ParseComposedKey(key)
var fp string
if err == nil && !isNode {
fp, err = catalogkey.ParseKeyPart(part)
}
switch {
case err != nil:
row.Unread = err.Error()
case isNode:
row.Module = "node"
default:
row.Module, row.FP = module, fp
}
if err == nil {
c, derr := nodeengineclient.DecodeComposition(raw)
switch {
case derr != nil:
row.Unread = derr.Error()
case c.Refused != "":
row.Refused = c.Refused
default:
if d, err := nodeengineclient.ComposedDigest(c); err != nil {
row.Unread = err.Error()
} else {
row.Digest = d
}
}
}
rows = append(rows, row)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
out[node] = rows
}
return out
}
@@ -1,380 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
const waitOnBuildNode = "build node"
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
return func(ask, why string) error {
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
if err != nil {
return err
}
if _, said, ok := buscalls.Unwrap(raw); !ok {
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
}
return nil
}
}
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
if run.Started != nil {
w.Since = *run.Started
}
return w
}
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
if r.say == nil {
return
}
for _, c := range readers {
w := waitOf(run, c)
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
if how != "" {
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
}
if err := r.say(event, w, id); err != nil {
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
}
}
}
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
func (r *compositionRuns) untaken(key, ask string) bool {
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
return err == nil && standing && held == ask
}
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
raw, rev, err := r.kv.get(boundKey)
if err != nil || raw == nil {
return "", 0, false, err
}
var v boundValue
_ = json.Unmarshal(raw, &v)
return v.Ask, rev, true, nil
}
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
rev, err := r.kv.arm(prefix+key, ask, ttl)
if err != nil {
return false, err
}
run, _, ok, err := r.read(key)
if err != nil {
return false, err
}
if ok && run.Ended == nil && run.Ask == ask {
return true, nil
}
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
return false, err
}
return false, nil
}
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
// ended, which moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
key, run, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
if key == "" {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
if err != nil {
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
}
if !armed {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
if r.untaken(key, e.Ask) {
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
return err
}
r.sayWaits(run, run.Readers, "ran")
}
return nil
}
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
// late one moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
key, _, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
armed := false
if key != "" {
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
return err
}
}
if !armed {
r.late("step-progress", e.Asker, e.Ask)
}
return nil
}
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
func (r *compositionRuns) late(signal, asker, ask string) {
if asker == runAsker && ask != "" {
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
}
}
// runOfAsk is the running run of an ask the assigner put, or none.
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
if asker != runAsker || ask == "" {
return "", meshwants.CompositionRun{}, nil
}
key, run, err := r.keyOfAsk(ask)
if err != nil || key == "" || run.Ended != nil {
return "", meshwants.CompositionRun{}, err
}
return key, run, nil
}
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
// ask and the bound. A run that ended already, or a key of none, takes nothing.
func (r *compositionRuns) expired(boundKey string) {
key, bound, setting := "", time.Duration(0), ""
switch {
case strings.HasPrefix(boundKey, runBoundPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
case strings.HasPrefix(boundKey, runUntakenPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
default:
return
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
return
}
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
// since, ends nothing.
other := runUntakenPrefix
if setting == "run-untaken-bound" {
other = runBoundPrefix
}
_, _, armed, err := r.boundOf(boundKey)
if err != nil {
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
return
}
held, _, standing, err := r.boundOf(other + key)
if err != nil {
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
return
}
if armed || (standing && held == run.Ask) {
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
return
}
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
if setting == "run-untaken-bound" {
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
}
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
r.logf("the composition run %s past its bound is not ended: %v", key, err)
}
}
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
}
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
}
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
if moved := r.moved.Load(); moved != nil && move {
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
(*moved)(run)
}
}
ended, err := r.end(key, ask, outcome, said, verdict)
if err != nil || !ended {
return ended, err
}
if untaken {
if run, _, ok, err := r.read(key); err == nil && ok {
r.sayWaits(run, run.Readers, "superseded")
}
}
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
}
return true, nil
}
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
seen := map[string]bool{}
var out []openReader
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
continue
}
for _, c := range run.Readers {
if c.Ended != nil && !c.Rerun {
continue // it ended passed already: only a reader marked rerun says its end again
}
if !seen[c.ID] {
seen[c.ID] = true
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
}
}
}
return out, nil
}
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
type openReader struct {
meshwants.CompositionReader
ask string
}
// keysOf are the keys of the candidate's runs that the reader reads.
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var out []string
prefix := keysPart(candidate) + "."
for _, key := range all {
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
continue
}
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
out = append(out, key)
}
}
slices.Sort(out) // by machine, as a step names its machines
return out, nil
}
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
// handed to the runs; one that cannot be written is given again by the bus.
func hearRunBounds(hub interface {
Handle(string, func(jetstream.Msg)) error
}, r *compositionRuns, logf func(string, ...any)) error {
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
var e nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardAskTaken(e), logf)
}); err != nil {
return err
}
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
var e nodebuildgo.StepProgress
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardProgress(e), logf)
})
}
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
if err != nil {
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
readers, err := r.readersOfOpen()
if err != nil {
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
return
}
for _, reader := range readers {
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
if reader.Ended != nil {
again = reader.ask
}
how := c.howOf(reader.Step, true)
if how == nil {
c.logf("the step %s of %s is not taken up again: it is no composition or validate step (%q)", reader.ID, reader.Module, reader.Step)
continue
}
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), how, again)
}
}
@@ -1,188 +0,0 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
// candidateKeyOf is the key the step's pipeline names its candidate by (A-226): a run on request's `run-<id>`, or for a branch
// pipeline the branch's catalog graph revision `<branch>.<revision>`, the branch escaped as a key part as the pipeline key does.
func candidateKeyOf(run meshwants.StepRun) (string, error) {
switch {
case run.Run != "":
return nec.RunKey(run.Run)
case run.Branch == "" || run.Revision <= 0:
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
}
return nec.RevisionKey(run.Branch, run.Revision) // built by the seat's client, never by hand (A-226b)
}
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
// naming none is the trunk's, trunk the first watched branch); a branch that is not watched takes the nodes placing the module on the trunk. In name order.
func machinesFor(placed map[string]map[string]string, module, branch, trunk string, watched bool) []string {
if branch == "" || !watched {
branch = trunk
}
var out []string
for node, b := range placed[module] {
if b == "" {
b = trunk
}
if b == branch {
out = append(out, node)
}
}
slices.Sort(out)
return out
}
// machinesOf are the machines of a composition step in the world's assignments.
func (w *world) machinesOf(module, branch string) []string {
return machinesFor(w.placedBranches(), module, branch, w.trunkName(), w.holdsBranch(branch))
}
// nodeInput is one machine's input: its bytes, or why the node gave none, or that it is away.
type nodeInput struct {
Node string
Body []byte
// Why is set when the node gave no input: its words, or the call's error.
Why string
// Away is a machine the watchman said away: it is not asked (§64).
Away bool
}
// compositionInputs asks the machines of a step for their inputs.
type compositionInputs struct {
machines func(module, branch string) []string
// callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to
// one node as `seat:<seat>.<verb>@<node>`, carrying the ask's id and stopping at its bound, and answers the verb's answer with
// the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the
// body's `call`.
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
// bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775).
bound time.Duration
}
// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no
// input, naming every call.
const (
composeAskBoundSetting = "compose-ask-bound"
defaultComposeAskBound = 10 * time.Second
)
// composeAskBound reads the setting compose-ask-bound (§781).
func composeAskBound() (time.Duration, error) {
d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per
// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id.
func composeAskID(stepAsk, node string) string { return stepAsk + "." + node }
// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step
// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until
// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput {
bound := c.bound
if bound <= 0 {
bound = defaultComposeAskBound
}
in := nodeInput{Node: node}
type reply struct {
raw json.RawMessage
err error
}
got := make(chan reply, 1)
go func() {
// The input is any size, so the verb sends it in chunks on the call token chosen here, and the file is the input (A-237b, A-237c).
var bits [16]byte
if _, err := rand.Read(bits[:]); err != nil {
got <- reply{nil, err}
return
}
raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node,
ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])})
got <- reply{raw, err}
}()
var raw json.RawMessage
select {
case <-ctx.Done():
in.Why = "its step's wait ended: " + context.Cause(ctx).Error()
return in
case r := <-got:
if r.err != nil {
in.Why = r.err.Error()
return in
}
raw = r.raw
}
var a ne.ComposeCandidateAnswer
// its numbers as written (json.Number), so the input kept is the engine's own bytes' values
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
if err := dec.Decode(&a); err != nil {
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
return in
}
switch {
case a.Verdict == nec.Unready:
in.Why = "unready: " + a.Refused
return in
case a.Chunks == 0 && len(a.Input) == 0:
in.Why = "the engine answered no composition input"
if a.Refused != "" {
in.Why += ": " + a.Refused
}
return in
}
var body []byte
if a.Chunks > 0 {
// The composition input is the file the chunks carried, the engine's own bytes (numbers as written).
var whole struct {
File string `json:"file"`
}
if err := json.Unmarshal(raw, &whole); err != nil {
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
return in
}
file, err := base64.StdEncoding.DecodeString(whole.File)
if err != nil || len(file) == 0 {
in.Why = fmt.Sprintf("the engine named %d chunks of composition input and the file received is empty or not base64: %v", a.Chunks, err)
return in
}
body = file
} else {
// An answer with the input inline (an engine before A-237b, read until the checker is removed, A-237c, G-6 §321): the object, as it was.
var err error
if body, err = json.Marshal(a.Input); err != nil {
in.Why = "its composition input cannot be encoded again: " + err.Error()
return in
}
}
in.Body = body
return in
}
@@ -1,168 +0,0 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"regexp"
"strings"
"sync"
"testing"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G A-226: the candidate a step asks is `run-<id>` for a run on request, else `<branch>.<revision>`.
func TestTheCandidateKeyOfAStep(t *testing.T) {
for _, c := range []struct {
run meshwants.StepRun
want string
fail bool
}{
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
{run: meshwants.StepRun{Branch: "main", Revision: 362}, want: "main.362"},
{run: meshwants.StepRun{Branch: "main"}, fail: true},
{run: meshwants.StepRun{Branch: "main", Revision: 100000000000}, fail: true}, // a number the client's key would read as a heads hash
{run: meshwants.StepRun{Run: "a.b"}, fail: true}, // a run id the client refuses
} {
got, err := candidateKeyOf(c.run)
if (err != nil) != c.fail || got != c.want {
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
}
// A-226b: the node-engine reads every key the step names with the client's one parser.
if !c.fail {
if _, err := nec.ParseCandidateKey(got); err != nil {
t.Errorf("%+v: the key %q is refused by the seat's parser: %v", c.run, got, err)
}
}
}
}
// A-252: the machines of a step are the nodes whose placement names its branch; none named is the trunk; an unwatched branch takes
// the trunk's nodes.
func TestTheMachinesOfAStep(t *testing.T) {
placed := map[string]map[string]string{"m": {"ace": "", "g14": "feat", "novox": "main", "shanks": "feat"}, "other": {"ace": ""}}
for _, c := range []struct {
branch string
watched bool
want string
}{
{"main", true, "ace novox"},
{"feat", true, "g14 shanks"},
{"gone", false, "ace novox"},
{"", true, "ace novox"},
} {
if got := strings.Join(machinesFor(placed, "m", c.branch, trunkBranch, c.watched), " "); got != c.want {
t.Errorf("branch %q watched %v: %q, want %q", c.branch, c.watched, got, c.want)
}
}
if got := machinesFor(placed, "none", "main", trunkBranch, true); len(got) != 0 {
t.Errorf("a module placed nowhere has machines %v", got)
}
}
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// the machines are asked side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
asks := map[string]process.KeptAsk{}
call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
mu.Lock()
asked[node], asks[node] = key+" "+string(raw), ask
mu.Unlock()
switch node {
case "g14":
return nil, errors.New("no responders")
case "novox":
return json.RawMessage(`{"candidate":"run-r1","node":"novox","verdict":"unready","refused":"starting","modules":[],"own":{},"took":1,"v":1}`), nil
}
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
// With a call token of 128 random bits (32 hex digits), the verb being marked chunks.
if k, body, _ := strings.Cut(asked["ace"], " "); k != "seat:node-engine.compose-candidate" ||
!regexp.MustCompile(`^\{"call":"[0-9a-f]{32}","candidate":"run-r1","input":true\}$`).MatchString(body) {
t.Errorf("asked %q", asked["ace"])
}
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
t.Errorf("ace: %+v", got[0])
}
// Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound.
if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" {
t.Errorf("the kept asks %+v", asks)
}
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
}
if _, was := asked["shanks"]; was || !got[3].Away {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
// A-237b: the composition input is any size, so a node names the chunks it sent and the assigner reads the file the chunked call
// received as the input, its numbers as written; a node that names chunks and sends no file is said so.
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
file := base64.StdEncoding.EncodeToString([]byte(input))
call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
_, node, _ := strings.Cut(key, "@")
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
if node == "g14" {
return json.RawMessage(answer + `}`), nil // names chunks, none received
}
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
if err != nil || len(got) != 2 {
t.Fatalf("%+v %v", got, err)
}
if got[0].Why != "" || string(got[0].Body) != input {
t.Errorf("ace: why %q, %d bytes, want the file received (%d bytes) as the input", got[0].Why, len(got[0].Body), len(input))
}
if !strings.Contains(got[1].Why, "chunks") || got[1].Body != nil {
t.Errorf("g14: %+v", got[1])
}
}
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
// is not asked; no machine is an error.
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, errors.New("no machine")
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }()
}
wg.Wait()
return out, nil
}
@@ -1,181 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
// could not run.
const (
// readsBound is how long a read is made again before the step could not run.
readsBound = 3 * time.Minute
// readsFirstWait and readsMaxWait bound the wait between two tries.
readsFirstWait = time.Second
readsMaxWait = 15 * time.Second
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
readsKept = 8
)
// readKey names one revision of one watched branch.
type readKey struct {
branch string
revision int
}
// readSlot is one revision's read: done closes when half or err is set.
type readSlot struct {
done chan struct{}
half mcc.Half
err error
}
// compositionReads shares the reads of composition steps.
type compositionReads struct {
mu sync.Mutex
revs map[readKey]*readSlot
order []readKey
seats map[string]json.RawMessage // by seat and digest: the definition, raw
}
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
func transient(err error, said string) bool {
text := said
if err != nil {
text = err.Error() + " " + said
}
text = strings.ToLower(text)
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
if strings.Contains(text, w) {
return true
}
}
return false
}
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
// a good answer, or the last fault.
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
start := r.now()
wait := readsFirstWait
for try := 1; ; try++ {
raw, err := r.call(key, body)
var inner json.RawMessage
var said string
if err == nil {
var ok bool
if inner, said, ok = buscalls.Unwrap(raw); ok {
return inner, "", nil
}
}
if !transient(err, said) {
return nil, said, err
}
if r.now().Sub(start)+wait > readsBound {
return nil, said, err
}
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
if serr := r.waitFor(ctx, wait); serr != nil {
return nil, said, serr
}
if wait *= 2; wait > readsMaxWait {
wait = readsMaxWait
}
}
}
// waitFor waits d, or until ctx ends.
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
if r.sleep != nil {
return r.sleep(ctx, d)
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
// failed is not kept, so the next step reads again.
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
c := r.reads
c.mu.Lock()
if c.revs == nil {
c.revs = map[readKey]*readSlot{}
}
if s, ok := c.revs[key]; ok {
c.mu.Unlock()
select {
case <-s.done:
if s.err == nil {
return s.half, nil
}
case <-ctx.Done():
return mcc.Half{}, ctx.Err()
}
// The read this step waited for failed: read it here (the failed slot is already dropped).
return r.half(ctx, key, read)
}
s := &readSlot{done: make(chan struct{})}
c.revs[key] = s
c.order = append(c.order, key)
for len(c.order) > readsKept {
delete(c.revs, c.order[0])
c.order = c.order[1:]
}
c.mu.Unlock()
s.half, s.err = read()
if s.err != nil {
c.mu.Lock()
if c.revs[key] == s {
delete(c.revs, key)
}
for i, k := range c.order {
if k == key {
c.order = append(c.order[:i], c.order[i+1:]...)
break
}
}
c.mu.Unlock()
}
close(s.done)
return s.half, s.err
}
// seatKept is the definition kept of a seat at a digest.
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
c.mu.Lock()
defer c.mu.Unlock()
d, ok := c.seats[seat+"@"+digest]
return d, ok
}
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
c.mu.Lock()
defer c.mu.Unlock()
if c.seats == nil {
c.seats = map[string]json.RawMessage{}
}
c.seats[seat+"@"+digest] = def
}
func unreadErr(what string, err error, said string) error {
if err != nil {
return fmt.Errorf("%s: %w", what, err)
}
return fmt.Errorf("%s: %s", what, said)
}
@@ -1,869 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests
// and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
// watchman said away (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
// asks are put on and the first of their step keys (§467, §481, §600, A-163).
runAsker = meshwants.Seat
// changeCase is the one case each request composes.
changeCase = "change"
// inputVerbKey is the verb a run ask's input is served on: the seat mesh-wants' input (§354).
inputVerbKey = "seat:" + meshwants.Seat + "." + meshwants.VerbInput
// entryVerbKey and seatDefinitionsKey are the mesh-catalog seat verbs the step reads.
entryVerbKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// compositionRun runs a composition step.
type compositionRun struct {
// runs are the composition runs of the state composition-runs (compositionruns.go).
runs *compositionRuns
inputs *compositionInputs
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
call func(key string, body any) (json.RawMessage, error)
// callFile asks a seat verb held once for the mesh whose answer comes in chunks, as a kept call (process.Conn.CallFileKept),
// answering the verb's answer with the bytes received as its member `file`, base64: the validate step's seat-list (H-A-443).
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
// trunk is the trunk's catalog half as the assigner holds it.
trunk func() (mcc.Half, bool)
// newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's
// watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across
// them (design 47 H-A-143, G §532). Nil: the step's revision alone.
newest func() map[string]mcc.Half
watched func(module string) []string
// assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists;
// assignment is the machine's assignments whole, which the case composes with those modules left out (H-A-148).
assigned func(node string) []string
assignment func(node string) (mma.Assignments, bool)
// findings is where the step raises the finding assignment-unlisted of each assignment it leaves out (H-A-149); the pass over
// the assignments clears it.
findings assign.Findings
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
running func(node, module string) (ne.Running, bool)
// away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow),
// so a durable on the watchman's streams has one hearer.
away func(machine string) bool
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
logf func(string, ...any)
}
// machineEnd is what one machine came to.
type machineEnd struct {
failed, couldNot string
warnings []string
// judged is set where the composer judged the machine, composing or not (A-467a).
judged bool
}
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
// compose, could not run when a machine could not be composed or was not judged, nothing to run when no node places the module on
// the branch (H-A-179), else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
nodes := r.inputs.machines(step.Module, step.Branch)
if len(nodes) == 0 { // a mover no machine holds on the branch has nothing to judge: nothing to run, passed (H-A-179, ADR 0682 §8)
return stepOutcome{Outcome: meshwants.StepEndedOutcomeNothingToRun,
Said: fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)}
}
reader := readerOf(id, step)
prep := &runPrep{r: r, step: step}
var ends []machineEnd
var keys []string
for _, node := range nodes {
if r.away != nil && r.away(node) {
ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}})
continue
}
join := r.runs.join
if step.Rerun { // a step asked again (§727)
join = r.runs.rejoin
}
j, err := join(reader, candidate, node, buildstep.NewID())
if err != nil {
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
continue
}
if j.started {
go r.start(ctx, j.key, j.run, prep, id)
}
keys = append(keys, j.key)
}
return r.read(ctx, step.Module, keys, ends)
}
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
// runs it reads and their asks (§723, §729).
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
setShared(ctx, r.runs.sharedOf(keys, nil))
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
runs := r.runs.await(ctx, keys)
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(module, runs[key]))
}
out := outcomeOf(ends)
out.Shared = r.runs.sharedOf(keys, runs)
return out
}
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
keys, err := r.runs.keysOf(candidate, id)
if err != nil {
return couldNotRun("the step's composition runs are not read: " + err.Error())
}
return r.read(ctx, step.Module, keys, nil)
}
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
// machine never passes (A-467a), each machine not judged named with why.
func outcomeOf(ends []machineEnd) stepOutcome {
var out stepOutcome
var failed, couldNot []string
judged := 0
for _, e := range ends {
if e.judged {
judged++
}
failed = appendSome(failed, e.failed)
couldNot = appendSome(couldNot, e.couldNot)
out.Warnings = append(out.Warnings, e.warnings...)
}
switch {
case len(failed) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
case judged == 0:
said := "no machine was judged"
if len(out.Warnings) == 0 {
said += ": the step has no machine"
} else {
said += ": " + strings.Join(out.Warnings, "; ")
}
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, said
default:
out.Outcome = meshwants.StepEndedOutcomePassed
}
return out
}
func appendSome(to []string, s string) []string {
if s == "" {
return to
}
return append(to, s)
}
func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
// case's manifests, and the composer program per machine. The case carries no verdicts: the step validate judged the
// manifests before the composition step (design 47 H-A-112, H-A-108).
type runPrep struct {
r *compositionRun
step meshwants.StepRun
once sync.Once
// read once
seats []json.RawMessage
manifests map[string]composer.Manifest
none map[string]string
programOf func(node string) (buildstep.RunProgram, string, map[string]string, error)
err error
}
func (p *runPrep) get(ctx context.Context) error {
p.once.Do(func() {
half, err := p.r.revision(ctx, p.step)
if err != nil {
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
return
}
if p.r.newest != nil && p.r.watched != nil {
half = acrossBranches(p.step, half, p.r.newest(), p.r.watched)
}
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
return
}
if p.manifests, p.none, p.err = caseOf(half); p.err != nil {
return
}
p.programOf, p.err = p.r.programs(ctx, half)
})
return p.err
}
// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`,
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) {
node, ask := run.Machine, run.Ask
end := func(said string) {
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
r.logf("the composition run %s is not ended: %v", key, err)
}
}
// The one set of unlisted assignments, read once at the run's start and carried on the run to the leaving-out, the finding and
// every step's judgement, the resume's too (design 47 H-A-152, H-A-153).
unlisted := r.unlistedOn(node)
if len(unlisted) > 0 {
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Unlisted = unlisted
return true
}); err != nil {
end(node + ": the run's unlisted assignments are not written: " + err.Error())
return
}
}
if err := prep.get(ctx); err != nil {
end(node + ": " + err.Error())
return
}
in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405)
if in.Why != "" {
end(node + " gave no composition input: " + in.Why)
return
}
program, engineFp, pins, err := prep.programOf(node)
if err != nil {
end(node + ": " + err.Error())
return
}
cs := composer.Case{Name: changeCase, Manifests: prep.manifests, None: prep.none}
if left, err := r.leftOut(node, unlisted); err != nil {
end(node + ": " + err.Error())
return
} else if left != nil {
cs.Assignments = left
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{cs}, Seats: prep.seats,
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
raw, err := json.Marshal(req)
if err != nil {
end(node + ": its request cannot be encoded: " + err.Error())
return
}
digest, err := r.keeper.keep(ask, raw)
if err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
end(node + ": its request is not kept: " + err.Error())
return
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
end(node + ": " + err.Error())
return
}
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
c.Input = digest
return true
}); err != nil {
end(node + ": the run's program and input are not written: " + err.Error())
return
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit},
Packages: pins} // the composer is built against its entry's pins, as every build (H-A-203, H-A-192)
if err := r.put(ctx, a); err != nil {
end(node + ": its run ask " + ask + " is not put: " + err.Error())
}
}
// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's
// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b).
const notTried = "not tried: the search reached its bound"
// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else
// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
//
// The run's unlisted are the modules the machine is assigned that no watched branch's newest revision lists, left out of its
// composition, as the run's start read them (H-A-152, H-A-153): each is said on every path, in the machine's failed or
// could-not-run words where one is set, else as a warning, never the machine's failure and never dropped in silence (H-A-143,
// H-A-144, H-A-148).
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
end := judgeVerdict(module, run)
for _, u := range run.Unlisted {
said := run.Machine + " is assigned " + u + ", which no watched branch lists (H-A-143)"
switch {
case end.failed != "":
end.failed += "; " + said
case end.couldNot != "":
end.couldNot += "; " + said
default:
end.warnings = append(end.warnings, said)
}
}
return end
}
// judgeVerdict is judgeRun before the unlisted modules are said.
func judgeVerdict(module string, run meshwants.CompositionRun) machineEnd {
node := run.Machine
if run.Ended == nil {
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
}
if run.Outcome != meshwants.CompositionRunOutcomePassed {
return machineEnd{couldNot: run.Said}
}
m, err := machineOf(run.Verdict)
if err != nil {
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
}
if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
}
why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")
switch {
case m.Blamed == nil && !m.Composes: // an old composer (§721)
return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}}
case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452)
return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why}
}
end := machineEnd{judged: true}
var own, others []string
for _, b := range m.Blamed {
switch {
case b.Module == module && b.Problem == notTried:
end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module)
case b.Module == module:
own = append(own, b.Problem)
default:
others = append(others, node+" holds "+b.Module+": "+b.Problem)
}
}
if len(own) > 0 {
end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), ""
}
end.warnings = others
return end
}
// machineOf is the composer's verdict of one machine as a run keeps it.
func machineOf(v map[string]any) (composer.Machine, error) {
var m composer.Machine
raw, err := json.Marshal(v)
if err == nil {
err = json.Unmarshal(raw, &m)
}
return m, err
}
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
func runVerdict(node string, answer []byte) (map[string]any, string) {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return nil, node + ": the composer answered no verdict: " + err.Error()
}
if v.V != composer.Version {
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
if !ok {
return nil, node + ": the composer's verdict names no such machine"
}
raw, err := json.Marshal(m)
var out map[string]any
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
}
return out, ""
}
return nil, node + ": the composer's verdict has no case " + changeCase
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
// it (compositionreads.go).
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
}
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
if body == nil {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(body, &a); err != nil {
return mcc.Half{}, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != run.Revision {
return mcc.Half{}, fmt.Errorf("%s answered no revision %d of %q", mcc.RevisionVerb, run.Revision, run.Branch)
}
if in.After == "" {
half.Revision = *a.Revision // the first page names what the revision was read from, its definitions included
}
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
in.After = a.Next
}
}
// seats are the definitions the composer is handed, raw. A revision that names definitions (A-261): the definition of each (seat, digest)
// pair, from mesh-catalog's seat-definitions where the trunk's digest is the revision's; a digest neither holds is an error naming the seat
// and the digest. A revision that names none (A-234's stand-in, A-467): every definition of the trunk seat list, read through
// seat-definitions; a list or a definition that cannot be read is an error naming it, never an empty list.
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
if len(defs) == 0 {
return r.trunkSeats(ctx)
}
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
def, err := r.seatAt(ctx, d.Seat, d.Digest)
if err != nil {
return nil, err
}
out = append(out, def)
}
return out, nil
}
// trunkSeats are the definitions of every seat of the trunk's seat list (seat-definitions with no seat names them with their digests).
func (r *compositionRun) trunkSeats(ctx context.Context) ([]json.RawMessage, error) {
what := "the trunk's seat list is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var list meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &list); err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
if len(list.Seats) == 0 {
return nil, errors.New(what + ": mesh-catalog's seat-definitions names no seat, so the composer has no seat list to judge by")
}
out := make([]json.RawMessage, 0, len(list.Seats))
for _, s := range list.Seats {
def, err := r.seatAt(ctx, s.Seat, s.Digest)
if err != nil {
return nil, err
}
out = append(out, def)
}
return out, nil
}
// seatAt is the definition of a seat at a digest, kept once read (a definition at a digest never changes).
func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json.RawMessage, error) {
if def, ok := r.reads.seatKept(seat, digest); ok {
return def, nil
}
what := "the definition of the seat " + seat + " is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: seat})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
if a.Entry == nil || a.Entry.Digest != digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", seat, digest)
}
def, err := json.Marshal(a.Entry.Definition)
if err != nil {
return nil, err
}
r.reads.keepSeat(seat, digest, def)
return def, nil
}
// caseOf is the case's manifests and none from the revision's entries: each manifest whole, and the revision's none words of each
// module it gives no fingerprint, whose build can never be made (design 47 G A-470j, A-470k). It sets no verdicts (H-A-112).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, error) {
manifests := map[string]composer.Manifest{}
var none map[string]string
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
manifests[name] = composer.Manifest{Raw: raw}
if e.Fp == "" && e.None != "" {
if none == nil {
none = map[string]string{}
}
none[name] = e.None
}
}
return manifests, none, nil
}
// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, H-A-144, G §532): each module's entry from
// the newest revision of the first of its watched branches, in its registration's order, that lists it, none for a module with no
// watched branches; the candidate's revision replaces its
// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest
// or the branch's is not held. The half keeps the candidate's revision, its definitions with it.
func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half {
halves := make(map[string]mcc.Half, len(newest)+1)
for b, h := range newest {
halves[b] = h
}
if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision {
halves[step.Branch] = candidate
} else {
mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)}
moved := []string{step.Module}
for _, m := range candidate.Revision.Movers {
moved = append(moved, m.Module)
}
for _, m := range moved {
if e, ok := candidate.Entries[m]; ok {
mix.Entries[m] = e
} else {
delete(mix.Entries, m)
}
}
halves[step.Branch] = mix
}
out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}}
for _, h := range halves {
for name := range h.Entries {
if _, done := out.Entries[name]; done {
continue
}
for _, b := range watched(name) { // none: no watched branch lists it, it is unlisted (H-A-144)
if e, ok := halves[b].Entries[name]; ok {
out.Entries[name] = e
break
}
}
}
}
return out
}
// unlistedOn is the one set of unlisted assignments of node (H-A-152): the modules the assignments place there that the newest
// revision of none of their watched branches lists, a module with no watched branches among them (H-A-143, H-A-144), by name.
// The candidate never enters it, so a module the change drops while still assigned stays in the composition and fails it as the
// change's own fault (ADR 0578 §2). Where the newest revisions are not followed yet nothing is unlisted: none while no branch's
// newest is held, and not a module one of whose watched branches' newest is not held yet.
func (r *compositionRun) unlistedOn(node string) []string {
if r.assigned == nil || r.newest == nil || r.watched == nil {
return nil
}
newest := r.newest()
if len(newest) == 0 {
return nil
}
var out []string
for _, m := range r.assigned(node) {
watched := r.watched(m)
if !listedOn(newest, watched, m) && !slices.ContainsFunc(watched, func(b string) bool { _, held := newest[b]; return !held }) {
out = append(out, m)
}
}
slices.Sort(out)
return slices.Compact(out)
}
// leftOut is the case's assignments of node with each module of the run's unlisted set left out, or nil where none is (the case
// then composes the input's): a stale assignment is a fault of the assignments, not of the change judged, so it is left out of the
// composition, said in the journal, raised as assignment-unlisted and, at the judgement, said as unlisted from the same set
// (design 47 H-A-148, H-A-149, H-A-152, H-A-143; ADR 0578 §2).
func (r *compositionRun) leftOut(node string, set []string) (map[string]json.RawMessage, error) {
if r.assignment == nil || len(set) == 0 {
return nil, nil
}
a, ok := r.assignment(node)
if !ok {
return nil, nil
}
var unlisted []string
for _, m := range a.Modules {
if slices.Contains(set, m.Module) {
unlisted = append(unlisted, m.Module)
}
}
if len(unlisted) == 0 {
return nil, nil
}
a.Modules = slices.DeleteFunc(slices.Clone(a.Modules), func(m mma.PlacedModule) bool { return slices.Contains(unlisted, m.Module) })
raw, err := mmac.EncodeAssignments(a)
if err != nil {
return nil, fmt.Errorf("its assignments with %s left out cannot be encoded: %w", strings.Join(unlisted, ", "), err)
}
r.logf("%s: the composition leaves out %s, assigned there and listed on no watched branch (H-A-148)", node, strings.Join(unlisted, ", "))
for _, m := range unlisted {
r.raiseUnlisted(node, m)
}
return map[string]json.RawMessage{node: raw}, nil
}
// raiseUnlisted writes the finding assignment-unlisted of module on node (H-A-149); one not written is said in the journal.
func (r *compositionRun) raiseUnlisted(node, module string) {
if r.findings.Put == nil {
return
}
now := r.now
if now == nil {
now = time.Now
}
f, err := assign.UnlistedFinding(node, module, now())
if err == nil {
err = r.findings.Put(f)
}
if err != nil {
r.logf("the finding %s is not written: %v", assign.UnlistedKey(node, module), err)
}
}
// listedOn says whether the newest revision of one of module's watched branches lists it (H-A-143, H-A-144).
func listedOn(newest map[string]mcc.Half, watched []string, module string) bool {
for _, b := range watched {
if _, ok := newest[b].Entries[module]; ok {
return true
}
}
return false
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(mapsKeys(entries)) {
for _, s := range entries[name].Serves {
// The catalog writes a served seat verb as `seat:<seat>.<verb>` (as the compose words above are made).
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), ne.Seat+".") {
return entries[name], true
}
}
}
return meshcatalog.CatalogEntry{}, false
}
func mapsKeys(m map[string]meshcatalog.CatalogEntry) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
// programOfEntry is the composer built from a node-engine entry: its repository, commit and directory, the composer's package.
func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
dir := e.Path
if dir == "" {
dir = "."
}
if e.Repository == "" || e.Commit == "" {
return buildstep.RunProgram{}, fmt.Errorf("the node-engine's entry %s names no repository or commit", e.Fp)
}
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// pinsOfEntry are the mesh packages' versions an entry pins, the ones it names with a version (H-A-203, H-A-183): the pins
// a run's program is built against, in the form the entry keeps and the builder sends. nil when it pins none.
func pinsOfEntry(e meshcatalog.CatalogEntry) map[string]string {
var out map[string]string
for path, version := range e.Packages {
if version == "" {
continue
}
if out == nil {
out = map[string]string{}
}
out[path] = version
}
return out
}
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
// `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, map[string]string, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, string, map[string]string, error) {
return p, engine.Fp, pinsOfEntry(engine), err
}, nil
}
var mu sync.Mutex
type built struct {
program buildstep.RunProgram
pins map[string]string
}
byFp := map[string]built{}
return func(node string) (buildstep.RunProgram, string, map[string]string, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
engine.Module, engine.Fp)
p, err := programOfEntry(engine)
return p, engine.Fp, pinsOfEntry(engine), err
}
fp := running.Fingerprint
mu.Lock()
b, ok := byFp[fp]
mu.Unlock()
if ok {
return b.program, fp, b.pins, nil
}
what := "the entry of the node-engine's build " + fp + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
if err != nil || body == nil {
return buildstep.RunProgram{}, fp, nil, unreadErr(what, err, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fp, nil, fmt.Errorf("%s: %w", what, err)
}
p, err := programOfEntry(a.Entry)
if err != nil {
return p, fp, nil, err
}
pins := pinsOfEntry(a.Entry)
mu.Lock()
byFp[fp] = built{program: p, pins: pins}
mu.Unlock()
return p, fp, pins, nil
}, nil
}
// trunkOf is the trunk's catalog half the assigner's views hold, trunk naming the first watched branch, or false while none is
// held.
func trunkOf(iv *catalogViews, trunk func() string) func() (mcc.Half, bool) {
return func() (mcc.Half, bool) {
if iv == nil || iv.h == nil {
return mcc.Half{}, false
}
h, ok := iv.h.views()[trunk()]
return h, ok
}
}
// runAskPublisher puts a run ask on the Go build seat's check queue under the assigner's name (`accept.check.assigner`, §600), its
// id the message id, so the stream takes it once.
func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildstep.StepAsk) error {
return func(ctx context.Context, a buildstep.StepAsk) error {
raw, err := buildstep.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, a.Asker)
if err != nil {
return err
}
_, err = js.Publish(ctx, subject, raw, jetstream.WithMsgID(a.ID))
return err
}
}
// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed.
func (iv *catalogViews) newestOf() func() map[string]mcc.Half {
if iv == nil {
return nil
}
return func() map[string]mcc.Half {
if iv.h == nil {
return map[string]mcc.Half{}
}
return iv.h.views()
}
}
// listed says whether one of module's watched branches lists it in the newest revision the views hold (H-A-149's clear).
func (iv *catalogViews) listed(module string) bool {
if iv == nil || iv.h == nil {
return false
}
return listedOn(iv.h.views(), iv.h.watchedOf(module), module)
}
// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed.
func (iv *catalogViews) watchedOf(module string) []string {
if iv == nil || iv.h == nil {
return nil
}
return iv.h.watchedOf(module)
}
@@ -1,807 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"maps"
"slices"
"strings"
"sync"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/assign"
)
const (
commitE1 = "1111111111111111111111111111111111111111"
commitE2 = "2222222222222222222222222222222222222222"
commitE3 = "3333333333333333333333333333333333333333"
)
// runStand is a composition run over stand-ins of the mesh-catalog seat, the machines' engines, the build seat and its step-ended.
type runStand struct {
mu sync.Mutex
asks map[string]buildstep.StepAsk // by node
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
runs *compositionRuns
puts int // the run asks put
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
noDefinitions bool
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
unjudged string
listFault error
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
verdict func(node string) (composer.Machine, bool)
}
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
engine := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
Manifest: map[string]any{"module": "engine"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25-" + fp, "git.novox.be/novox/x/go": ""}}
}
revisionCommit := commitE1
if revisionEngine != "src2:e1" {
revisionCommit = commitE3
}
pages := map[string]meshcatalog.RevisionAnswer{
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
switch key {
case mcc.RevisionVerb:
var in meshcatalog.RevisionInput
_ = json.Unmarshal(raw, &in)
if in.Revision != 7 || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
page := pages[in.After]
if s.noDefinitions && page.Revision != nil {
rev := *page.Revision
rev.Definitions = nil
page.Revision = &rev
}
return json.Marshal(page)
case seatDefinitionsKey:
var sin meshcatalog.SeatDefinitionsInput
_ = json.Unmarshal(raw, &sin)
if sin.Seat == "" {
if s.listFault != nil {
return nil, s.listFault
}
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Seats: s.trunkSeats})
}
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
case entryVerbKey:
var in meshcatalog.EntryInput
_ = json.Unmarshal(raw, &in)
s.mu.Lock()
s.entries = append(s.entries, in.Fingerprint)
s.mu.Unlock()
commit := map[string]string{"src2:e1": commitE1, "src2:e2": commitE2}[in.Fingerprint]
return json.Marshal(meshcatalog.EntryAnswer{Entry: engine(in.Fingerprint, commit)})
}
return nil, errors.New("no such verb " + key)
}
engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) {
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
away := func(machine string) bool { return machine == "shanks" }
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
s.r = &compositionRun{runs: s.runs,
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
running: func(node, module string) (ne.Running, bool) {
fp, ok := running[node]
return ne.Running{Node: node, Module: module, Fingerprint: fp}, ok && module == "engine"
},
put: func(_ context.Context, a buildstep.StepAsk) error {
body, err := keeper.body(a.Run.Input.Digest)
if err != nil {
return err
}
req, err := composer.Decode(body)
if err != nil {
return err
}
node := req.Machines[0]
s.mu.Lock()
s.asks[node], s.requests[node] = a, req
s.mu.Unlock()
m := composer.Machine{Composes: true}
override, set := composer.Machine{}, false
if s.verdict != nil {
override, set = s.verdict(node)
}
if set {
m = override
} else if node == s.unjudged {
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
} else if node == "g14" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
s.mu.Lock()
s.puts++
s.mu.Unlock()
go func() {
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Error(err)
}
}()
return nil
},
}
return s
}
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
// depth compose with the one machine and one case `change`, its manifests and seats from the step's revision, no verdicts (H-A-112); the
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
// trunk's, or the revision's for a machine whose node-engine is not heard running (A-451); the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
out := s.r.run(context.Background(), "ask-step", step)
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
out.Warnings[1] != "g14: the composer names no culprits" {
t.Fatalf("the step came to %+v", out)
}
if len(s.asks) != 2 {
t.Fatalf("asked %d runs, want one per machine not away", len(s.asks))
}
a, req := s.asks["ace"], s.requests["ace"]
if a.Kind != buildstep.AskCheck || a.Language != buildstep.AskGo || a.Asker != runAsker || a.Run == nil || len(a.Run.Heads) != 0 ||
a.Run.Input.Verb != "seat:mesh-wants.input" || a.Run.Program != (buildstep.RunProgram{Repository: "novox/mesh-host", Commit: commitE1, Dir: ".", Package: composer.Package}) {
t.Errorf("the run ask of ace %+v %+v", a, a.Run)
}
if s.asks["g14"].Run.Program.Commit != commitE2 {
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
}
// Each composer is built against the pins of the node-engine entry it is built from, a pin with no version left out
// (H-A-203, H-A-192).
for node, fp := range map[string]string{"ace": "src2:e1", "g14": "src2:e2"} {
if got := s.asks[node].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25-"+fp {
t.Errorf("%s's composer is built against the pins %v; want its entry's", node, got)
}
}
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
t.Errorf("the request of ace %+v", req)
}
c := req.Cases[0]
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" {
t.Errorf("the case %+v", c.Manifests)
}
if strings.Join(s.entries, " ") == "" {
t.Error("no node-engine entry is read through entry")
}
// The revision's node-engine differs from the trunk's: that build composes every machine, and no entry is asked.
changed := newRunStand(t, "src2:e3", "src2:e1", "d1")
if out := changed.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed {
t.Fatalf("the step came to %+v", out)
}
if changed.asks["ace"].Run.Program.Commit != commitE3 || changed.asks["g14"].Run.Program.Commit != commitE3 || len(changed.entries) != 0 {
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
}
// No running of the node-engine is heard on g14: its composer is the revision's entry, and no entry is asked for it (A-451).
unheard := newRunStand(t, "src2:e1", "src2:e1", "d1")
unheard.r.running = func(node, module string) (ne.Running, bool) {
return ne.Running{Node: node, Module: module, Fingerprint: "src2:e1"}, node == "ace" && module == "engine"
}
if out := unheard.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") {
t.Fatalf("the step with g14's node-engine not heard came to %+v", out)
}
if unheard.asks["g14"].Run == nil || unheard.asks["g14"].Run.Program.Commit != commitE1 {
t.Errorf("g14's composer with no running heard is %+v, want the revision's entry", unheard.asks["g14"].Run)
}
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "s1") || !strings.Contains(out.Said, "d1") || len(other.asks) != 0 {
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
}
// A revision naming no definition (A-234's stand-in): the composer is handed the trunk seat list's definitions, read through
// seat-definitions (A-467); a list that cannot be read, or names no seat, ends the step could not run, naming it, with no ask put.
bare := newRunStand(t, "src2:e1", "src2:e1", "d1")
bare.noDefinitions, bare.trunkSeats = true, []meshcatalog.SeatDefinitionsAnswerSeatsItem{{Seat: "s1", Digest: "d1"}}
if out := bare.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
len(bare.requests["ace"].Seats) != 1 || string(bare.requests["ace"].Seats[0]) != `{"seat":"s1"}` {
t.Errorf("a revision with no definitions came to %+v, seats %v", out, bare.requests["ace"].Seats)
}
for name, fault := range map[string]error{"unreadable": errors.New("no responder"), "empty": nil} {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.noDefinitions, s.listFault = true, fault
if out := s.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "seat list is not read") || len(s.asks) != 0 {
t.Errorf("the %s seat list came to %+v, %d asks", name, out, len(s.asks))
}
}
// A machine the composer did not judge is could not run, naming why: a step that judged nothing never passes (A-467).
nj := newRunStand(t, "src2:e1", "src2:e1", "d1")
nj.unjudged = "g14"
if out := nj.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "g14 was not judged: the composer was handed no seat list") {
t.Errorf("a machine not judged came to %+v", out)
}
// No machine judged: every machine away, or no machine, never passes; each is named with why (A-467a).
for name, machines := range map[string][]string{"every machine away": {"shanks"}, "no machine": {}} { // none placed: the inputs say so already
want := "no machine was judged"
if len(machines) == 0 {
want = "has no machine to compose"
}
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
none.r.inputs.machines = func(string, string) []string { return machines }
outcome := meshwants.StepEndedOutcomeCouldNotRun
if len(machines) == 0 { // a mover no machine holds: nothing to run, saying why (H-A-179)
outcome = meshwants.StepEndedOutcomeNothingToRun
}
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != outcome ||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
}
}
// Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in
// the machine's reason (H-A-143, §532).
across := newRunStand(t, "src2:e1", "src2:e1", "d1")
across.r.newest = func() map[string]mcc.Half {
return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2},
Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}}
}
across.r.watched = func(module string) []string {
if module == "checker" || module == "lidarr" { // no registration names them; lidarr is on main's revision all the same (H-A-144)
return nil
}
return []string{"main", "master"}
}
across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker", "lidarr"} }
if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
!strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") ||
!strings.Contains(out.Said, "g14 is assigned lidarr, which no watched branch lists (H-A-143)") {
t.Errorf("the step across the watched branches came to %+v", out)
}
if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 4 ||
c.Manifests["lidarr"].Raw != nil {
t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests)))
}
if c := across.requests["ace"].Cases[0]; c.Assignments != nil {
t.Errorf("with no assignments held the case composes the input's, it carries %v", c.Assignments)
}
// A stale assignment is left out of the case's assignments as well, said, and the machine composes; a listed module stays in
// (H-A-148, ADR 0578 §2). The one set is the watched branches' newest revisions', carried on the run (H-A-152, H-A-153): a module
// the candidate drops while the newest still lists it stays in, the change's own fault, and while no newest revision is followed
// nothing is left out.
dropsWeb := func() map[string]mcc.Half {
h := across.r.newest()
h["main"] = mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: "main", Revision: 6},
Entries: map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Manifest: map[string]any{"module": "m"}},
"web": {Module: "web", Fp: "w1", Manifest: map[string]any{"module": "web"}}}}
return h
}
for _, tc := range []struct {
name string
assigned []string
newest func() map[string]mcc.Half // nil: across's
want []string // the case's modules of ace; nil: the input's assignments
}{
{"a stale assignment left out and said, the machine composes", []string{"m", "f1-circuits", "checker"}, nil, []string{"m", "f1-circuits"}},
{"listed modules alone: the input's assignments", []string{"m", "f1-circuits"}, nil, nil},
{"a module the candidate drops, the newest lists: kept", []string{"m", "f1-circuits", "web"}, dropsWeb, nil},
{"no newest revision followed: nothing left out", []string{"m", "checker"}, func() map[string]mcc.Half { return map[string]mcc.Half{} }, nil},
} {
left := newRunStand(t, "src2:e1", "src2:e1", "d1")
left.r.newest, left.r.watched = across.r.newest, across.r.watched
if tc.newest != nil {
left.r.newest = tc.newest
}
left.r.assigned = func(string) []string { return tc.assigned }
left.r.assignment = func(node string) (mma.Assignments, bool) {
a := mma.Assignments{Node: node}
for _, m := range tc.assigned {
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
}
return a, true
}
left.verdict = func(string) (composer.Machine, bool) { return composer.Machine{Composes: true}, true }
raised := map[string][]byte{}
left.r.findings = assign.Findings{Put: putInto(raised)}
out := left.r.run(context.Background(), "ask-step", step)
// H-A-149: the step that leaves an assignment out raises assignment-unlisted, scope delivery, id <node>.<module>, a warning.
var f mcseat.Finding
_ = json.Unmarshal(raised[assign.UnlistedKey("ace", "checker")], &f)
if want := tc.want != nil; want != (f.Kind == assign.KindAssignmentUnlisted && f.Scope == "delivery" && f.ID == "ace.checker" &&
f.Severity == mcseat.FindingSeverityWarning) || (!want && len(raised) != 0) {
t.Errorf("%s: raised %v", tc.name, raised)
}
said := "ace is assigned checker, which no watched branch lists (H-A-143)"
if out.Outcome != meshwants.StepEndedOutcomePassed || (tc.want != nil) != slices.Contains(out.Warnings, said) {
t.Errorf("%s: the step came to %+v", tc.name, out)
}
if run, _, _, _ := left.runs.read(runKeyOf("main.7", "ace")); (tc.want != nil) != slices.Equal(run.Unlisted, []string{"checker"}) ||
(tc.want == nil && run.Unlisted != nil) {
t.Errorf("%s: the run carries the unlisted %v", tc.name, run.Unlisted)
}
raw, ok := left.requests["ace"].Cases[0].Assignments["ace"]
if tc.want == nil {
if ok {
t.Errorf("%s: the case carries assignments %s", tc.name, raw)
}
continue
}
a, err := mmac.DecodeAssignments(raw)
var got []string
for _, m := range a.Modules {
got = append(got, m.Module)
}
if err != nil || !slices.Equal(got, tc.want) {
t.Errorf("%s: the case composes ace with %v (%v), want %v", tc.name, got, err, tc.want)
}
}
// At the inputs state's cap the ask is refused, naming the store (§354).
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
if out := full.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "the state inputs is at its cap") || len(full.asks) != 0 {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
for _, tc := range []struct {
name string
faults int // consecutive faults of the `revision` read before it answers
fault error // the fault
steps int
want string // the outcome of the last step
revisionCalls int
waits int
}{
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
} {
t.Run(tc.name, func(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
inner := s.r.call
var mu sync.Mutex
calls, faults := 0, tc.faults
s.r.call = func(key string, body any) (json.RawMessage, error) {
if key == mcc.RevisionVerb {
mu.Lock()
calls++
f := faults > 0
faults--
mu.Unlock()
if f {
return nil, tc.fault
}
}
return inner(key, body)
}
clock := time.Unix(1000, 0)
var waits []time.Duration
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
s.r.sleep = func(_ context.Context, d time.Duration) error {
mu.Lock()
defer mu.Unlock()
waits = append(waits, d)
clock = clock.Add(d)
return nil
}
var wg sync.WaitGroup
outs := make([]stepOutcome, tc.steps)
for i := range outs {
wg.Add(1)
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
}
wg.Wait()
for _, o := range outs {
if o.Outcome != tc.want {
t.Fatalf("a step came to %+v, want %s", o, tc.want)
}
}
if tc.waits == -1 { // never answers: bounded, growing waits
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
}
var total time.Duration
for _, w := range waits {
total += w
}
if total > readsBound {
t.Errorf("read again for %s, bound %s", total, readsBound)
}
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
t.Errorf("the step said %q", outs[0].Said)
}
return
}
if calls != tc.revisionCalls || len(waits) != tc.waits {
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
}
})
}
}
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
// own runs, and a run that could not run is never read again.
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var wg sync.WaitGroup
outs := make([]stepOutcome, 3)
for i, m := range []string{"m", "m2", "m3"} {
wg.Add(1)
go func() {
defer wg.Done()
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
}()
}
wg.Wait()
for i, o := range outs {
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
t.Errorf("step %d names the runs %+v", i, o.Shared)
}
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
t.Errorf("step %d came to %+v, want both machines read", i, o)
}
}
if s.puts != 2 {
t.Fatalf("%d run asks put, want one per machine", s.puts)
}
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
}
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
}
// Another candidate (a run on request) starts its own runs.
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
}
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
put, refused := f.r.put, 0
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
if refused == 0 {
refused++
return errors.New("no stream")
}
return put(ctx, a)
}
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
// try is could not run for its own step; a run that could not run is could not run.
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
at := time.Now()
runOf := func(verdict string) meshwants.CompositionRun {
var v map[string]any
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
t.Fatal(err)
}
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
}
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
for _, tc := range []struct {
name, module string
run meshwants.CompositionRun
failed, couldNo string
warnings []string
unlisted []string
}{
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil},
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil},
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil},
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil},
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil},
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil},
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
[]string{"ace holds a: not tried: the search reached its bound"}, nil},
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil},
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil},
// H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes.
{"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`),
"ace does not compose and blames no module: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
{"does not compose, blames a module, an unlisted module", "a", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[{"module":"a","problem":"nothing provides \"photos\""}]}`),
"ace holds a: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
{"another module blamed, an unlisted module", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"x"}]}`), "", "",
[]string{"ace holds a: x", "ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
{"could not run, an unlisted module", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"},
"", "ace: gone; ace is assigned photos, which no watched branch lists (H-A-143)", nil, []string{"photos"}},
{"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "",
[]string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
} {
t.Run(tc.name, func(t *testing.T) {
run := tc.run
run.Unlisted = tc.unlisted // the set the run's start wrote (H-A-153)
e := judgeRun(tc.module, run)
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
}
})
}
}
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
// step passed keeps its verdict.
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var mu sync.Mutex
fixed := false
s.verdict = func(node string) (composer.Machine, bool) {
mu.Lock()
defer mu.Unlock()
m := composer.Machine{Composes: true}
m.Blamed = []struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{}
if node == "g14" && !fixed {
m.Composes, m.Reason = false, "two modules own one path"
m.Blamed = append(m.Blamed, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"a", "/etc/x"}, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"d", "/etc/x"})
}
return m, true
}
ends := make(chan meshwants.StepEnded, 16)
msgIDs := map[string]string{} // the message id of each step's latest end
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
say: func(event string, body any, msgID string) error {
if e, ok := body.(meshwants.StepEnded); ok {
mu.Lock()
msgIDs[e.ID] = msgID
mu.Unlock()
ends <- e
}
return nil
}}
move := steps.move(context.Background())
s.runs.moved.Store(&move)
step := func(m string, rerun bool) []byte {
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
return raw
}
got := map[string]meshwants.StepEnded{}
for _, m := range []string{"a", "b", "d"} {
pullOne(steps, "ask-"+m, step(m, false))
e := <-ends
got[e.Module] = e
}
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the first ends: %+v", got)
}
former := got["d"].Shared[1].Ask
mu.Lock()
fixed = true
mu.Unlock()
pullOne(steps, "ask-a2", step("a", true))
moved := map[string]meshwants.StepEnded{}
for range 2 {
select {
case e := <-ends:
moved[e.ID] = e
case <-time.After(5 * time.Second):
t.Fatalf("the ends after the rerun: %+v", moved)
}
}
a, d := moved["ask-a2"], moved["ask-d"]
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
}
mu.Lock()
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
t.Errorf("the message ids of the ends: %v", msgIDs)
}
mu.Unlock()
if s.puts != 3 {
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
}
select {
case e := <-ends:
t.Errorf("a reader that passed moved: %+v", e)
case <-time.After(50 * time.Millisecond):
}
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
marks := map[string]bool{}
for _, c := range run.Readers {
marks[c.ID] = c.Rerun
}
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
t.Errorf("the new run's readers %+v", run.Readers)
}
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
deadline := time.After(5 * time.Second)
for {
wake := s.runs.wake()
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
break
}
select {
case <-wake:
case <-deadline:
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
}
}
}
// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its
// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the
// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision.
func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) {
entry := func(module, fp string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}}
}
half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half {
h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: branch}}},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, m := range movers {
h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m})
}
for _, e := range entries {
h.Entries[e.Module] = e
}
return h
}
newest := map[string]mcc.Half{
"main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")),
"master": half("master", 9, nil, entry("f1-circuits", "f1")),
"nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")),
}
registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}}
registrations["engine"], registrations["a"] = []string{"main"}, []string{"main"}
watched := func(module string) []string { return registrations[module] } // a module no registration names has none (H-A-144)
for _, tc := range []struct {
name string
step meshwants.StepRun
candidate mcc.Half
newest map[string]mcc.Half
want map[string]string // module -> fingerprint
}{
{"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest,
map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}},
{"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2},
half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest,
map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}},
{"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1},
half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]},
map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}},
{"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}},
{"a module no registration names is not composed", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
half("main", 5, nil, entry("a", "a2"), entry("checker", "c1")), nil, map[string]string{"a": "a2"}},
} {
t.Run(tc.name, func(t *testing.T) {
got := acrossBranches(tc.step, tc.candidate, tc.newest, watched)
fps := map[string]string{}
for name, e := range got.Entries {
fps[name] = e.Fp
}
if !maps.Equal(fps, tc.want) {
t.Errorf("composed from %v, want %v", fps, tc.want)
}
if got.Revision.Revision != tc.candidate.Revision.Revision || len(got.Revision.Definitions) != 1 || got.Revision.Definitions[0].Digest != tc.step.Branch {
t.Errorf("the half keeps revision %d with definitions %v, want the candidate's %d with its own", got.Revision.Revision, got.Revision.Definitions,
tc.candidate.Revision.Revision)
}
})
}
}
// Design 47 H-A-149: the pass over the assignments clears assignment-unlisted when the assignment is gone or one of the module's
// watched branches lists it again, and keeps it while it stands; another finding is left alone.
func TestThePassClearsAnUnlistedAssignmentsFinding(t *testing.T) {
for _, tc := range []struct {
name string
assigned []string // novox's modules
listed bool
kept bool
}{
{"still assigned and listed nowhere: kept", []string{"checker"}, false, true},
{"the assignment gone: cleared", []string{"web"}, false, false},
{"listed again: cleared", []string{"checker"}, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
w := newWorld()
a := mma.Assignments{V: keys.Version, Node: "novox"}
for _, m := range tc.assigned {
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
}
putAssignments(w, a)
findings := map[string][]byte{}
f, _ := assign.UnlistedFinding("novox", "checker", time.Now())
_ = putInto(findings)(f)
other, _ := assign.UnplacedFinding("g14", time.Now())
_ = putInto(findings)(other)
p := &passer{w: w, logf: t.Logf, listed: func(string) bool { return tc.listed }, findings: assign.Findings{Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil },
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(findings)), nil }}}
p.clearUnlisted()
if _, ok := findings[assign.UnlistedKey("novox", "checker")]; ok != tc.kept {
t.Errorf("the finding kept %v, want %v", ok, tc.kept)
}
if _, ok := findings[assign.UnplacedKey("g14")]; !ok {
t.Error("another finding is cleared")
}
})
}
}
@@ -1,916 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
)
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
const (
// runsSwept is how many of the oldest ended keys each new run judges.
runsSwept = 20
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
runBoundPrefix = "bound."
runUntakenPrefix = "untaken."
)
// runKeyOf is the key of a candidate's composition run of a machine (§719).
func runKeyOf(candidate, machine string) string {
return keysPart(candidate) + "." + keys.EscapeName(machine)
}
// keysPart is a name as a key's part.
func keysPart(name string) string { return keys.EscapeName(name) }
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
func isRunKey(key string) bool {
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
}
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
type runSettings struct {
// capBytes is composition-runs-cap-mib in bytes.
capBytes int64
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
kept, stepBound, untaken time.Duration
// blameTries is blame-tries, carried in each run's request (§721).
blameTries int
}
// defaultRunSettings are the settings' defaults, as the manifest gives them.
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
func runSettingsOf(getenv func(string) string) (runSettings, error) {
s := defaultRunSettings
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
if err != nil {
return s, err
}
if mib > 0 {
s.capBytes = int64(mib) << 20
}
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
if err != nil {
return s, err
}
if tries > 0 {
s.blameTries = tries
}
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
d, err := durationFile(getenv(env))
if err != nil {
return s, err
}
if d > 0 {
*to = d
}
}
return s, nil
}
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
func durationFile(path string) (time.Duration, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
}
return d, nil
}
// errRunChanged is a compare-and-set of a run's key that met another write.
var errRunChanged = errors.New("the composition run's key changed")
// runsKV is the state composition-runs: the bus bucket, or a test's.
type runsKV interface {
// get is a key's value and revision; nil for a key not there.
get(key string) ([]byte, uint64, error)
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
create(key string, value []byte) (uint64, error)
update(key string, value []byte, revision uint64) (uint64, error)
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
// of the state is made from a value read before its latest change.
delAt(key string, revision uint64) error
keys() ([]string, error)
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
// answers the revision written.
arm(key, ask string, ttl time.Duration) (uint64, error)
// used is how many bytes the state holds.
used() (int64, error)
}
// busRuns is runsKV over the bus bucket.
type busRuns struct {
ctx context.Context
js jetstream.JetStream
kv jetstream.KeyValue
}
func (b busRuns) get(key string) ([]byte, uint64, error) {
e, err := b.kv.Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (b busRuns) create(key string, value []byte) (uint64, error) {
rev, err := b.kv.Create(b.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := b.kv.Update(b.ctx, key, value, revision)
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) delAt(key string, revision uint64) error {
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return errRunChanged
}
return err
}
func (b busRuns) keys() ([]string, error) {
l, err := b.kv.ListKeys(b.ctx)
if err != nil {
return nil, err
}
var out []string
for k := range l.Keys() {
out = append(out, k)
}
return out, nil
}
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
type boundValue struct {
Bound string `json:"bound"`
Ask string `json:"ask"`
}
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
// takes its time to live with it, so the expiry is heard only ttl after the last write.
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
if b.js == nil {
return 0, errors.New("no bus to arm a bound on")
}
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
if err != nil {
return 0, err
}
subject := "$KV." + b.kv.Bucket() + "." + key
for {
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
if err != nil {
return 0, err
}
var last uint64
m, err := st.GetLastMsgForSubject(b.ctx, subject)
switch {
case err == nil:
last = m.Sequence
case !errors.Is(err, jetstream.ErrMsgNotFound):
return 0, err
}
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
continue
}
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
}
func (b busRuns) used() (int64, error) {
st, err := b.kv.Status(b.ctx)
if err != nil {
return 0, err
}
return int64(st.Bytes()), nil
}
// compositionRuns keeps the composition runs in the state composition-runs.
type compositionRuns struct {
kv runsKV
set runSettings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
changed chan struct{}
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
// being written at that moment, so every lookup takes these beside it (keys).
seen map[string]bool
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
// nil says or drops nothing.
say publisher
drop func(ask, why string) error
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
// Set once the steps are served, after the hub starts.
moved atomic.Pointer[func(run meshwants.CompositionRun)]
}
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
}
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
func (r *compositionRuns) heard() {
r.mu.Lock()
close(r.changed)
r.changed = make(chan struct{})
r.mu.Unlock()
}
// saw keeps a key the state's watch handed: standing, or deleted or expired.
func (r *compositionRuns) saw(key string, standing bool) {
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]bool{}
}
if standing {
r.seen[key] = true
} else {
delete(r.seen, key)
}
}
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
// is skipped by its reader.
func (r *compositionRuns) keys() ([]string, error) {
all, err := r.kv.keys()
if err != nil {
return nil, err
}
r.mu.Lock()
for key := range r.seen {
all = append(all, key)
}
r.mu.Unlock()
slices.Sort(all)
return slices.Compact(all), nil
}
// wake is closed at the next change heard.
func (r *compositionRuns) wake() <-chan struct{} {
r.mu.Lock()
defer r.mu.Unlock()
return r.changed
}
// read is a run's value and revision, or false.
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
raw, rev, err := r.kv.get(key)
if err != nil || raw == nil {
return meshwants.CompositionRun{}, 0, false, err
}
var run meshwants.CompositionRun
if err := json.Unmarshal(raw, &run); err != nil {
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
}
return run, rev, true, nil
}
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
// It answers the value written or left, and false for a key not there.
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok {
return run, ok, err
}
if !f(&run) {
return run, true, nil
}
raw, err := json.Marshal(run)
if err != nil {
return run, true, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return run, true, err
}
r.heard()
return run, true, nil
}
}
// fill is the state's fill against its cap.
func (r *compositionRuns) fill() (meshwants.Fill, error) {
at := r.now().UTC()
used, err := r.kv.used()
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
}
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
// value as it stood.
type joined struct {
key string
started bool
run meshwants.CompositionRun
}
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
// cap no run is started, an error naming the fill.
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
return r.joinAt(runKeyOf(candidate, machine), reader, candidate, machine, ask)
}
// joinAt is join of the run under key: a machine's composition run, or a revision's validate run with no machine (H-A-108).
func (r *compositionRuns) joinAt(key string, reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
if hasReader(run.Readers, reader.ID) {
return joined{key: key, run: run}, nil
}
run.Readers = append(run.Readers, reader)
raw, err := json.Marshal(run)
if err != nil {
return joined{}, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
if run.Ended == nil && r.untaken(key, run.Ask) {
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
Readers: []meshwants.CompositionReader{reader}}
raw, err := json.Marshal(fresh)
if err != nil {
return joined{}, err
}
if ok {
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
} else {
_, err = r.kv.create(key, raw)
}
if errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
func (r *compositionRuns) room() error {
if err := r.sweep(); err != nil {
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
}
f, err := r.fill()
if err != nil {
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
}
if f.Bytes >= f.Cap {
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
f.Cap>>20, f.Bytes)
}
return nil
}
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
func (r *compositionRuns) sweep() error {
all, err := r.keys()
if err != nil {
return err
}
type endedAt struct {
key string
at time.Time
}
var ended []endedAt
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended == nil {
continue
}
ended = append(ended, endedAt{key, *run.Ended})
}
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
if len(ended) > runsSwept {
ended = ended[:runsSwept]
}
var errs []error
for _, e := range ended {
if r.now().Sub(e.at) >= r.set.kept {
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
}))
}
}
return errors.Join(errs...)
}
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok || !gone(run) {
return err
}
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
return err
}
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return err
}
r.heard()
return nil
}
}
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
var errs []error
for _, prefix := range prefixes {
for {
run, _, ok, err := r.read(key)
if err != nil {
errs = append(errs, err)
break
}
if ok && run.Ask != ask {
break // a newer run holds the key: its bounds are its own
}
held, rev, standing, err := r.boundOf(prefix + key)
if err != nil || !standing {
errs = append(errs, err)
break
}
if held != ask {
break // the bound of another ask (A-470n): not this run's to delete
}
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
errs = append(errs, err)
}
break
}
}
return errors.Join(errs...)
}
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
keys, err := r.keysOfStep(step, id)
if err != nil {
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
return
}
at = at.UTC()
for _, key := range keys {
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
if i < 0 {
return false
}
c.Readers[i].Ended = &at
return true
}); err != nil {
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
}
}
}
// keysOfStep are the keys of the runs the step asked under id reads: its revision's validate run for a validate step, its
// candidate's runs naming it for a composition step.
func (r *compositionRuns) keysOfStep(step meshwants.StepRun, id string) ([]string, error) {
if step.Step == meshwants.StepRunStepValidate {
key, err := validateKeyOf(step)
if err != nil {
return nil, err
}
return []string{key}, nil
}
candidate, err := candidateKeyOf(step)
if err != nil {
return nil, err
}
return r.keysOf(candidate, id)
}
// hasReader is whether readers holds the step asked under id.
func hasReader(readers []meshwants.CompositionReader, id string) bool {
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
}
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
}
// stepRunOf is a reader's StepRun again, for its step-ended.
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
}
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
w, err := kv.WatchAll(ctx)
if err != nil {
return err
}
go func() {
defer func() { _ = w.Stop() }()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil {
if ready != nil {
ready()
ready = nil
}
continue
}
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
expired(e.Key())
}
r.heard()
}
}
}()
return nil
}
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
// false when the key holds another run or the run has ended already.
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
ended := false
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
at := r.now().UTC()
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
ended = true
return true
})
return ended, err
}
// keyOfAsk is the key of the run whose ask is ask, or none.
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
all, err := r.keys()
if err != nil {
return "", meshwants.CompositionRun{}, err
}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err == nil && ok && run.Ask == ask {
return key, run, nil
}
}
return "", meshwants.CompositionRun{}, nil
}
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
// names is another asker's, or of a run gone, and changes nothing.
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
if e.Asker != runAsker || e.Ask == "" {
return nil
}
key, run, err := r.keyOfAsk(e.Ask)
if err != nil || key == "" {
return err
}
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
return nil
}
node := run.Machine
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
var verdict map[string]any
if e.Outcome != buildstep.OutcomePassed {
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
said = node + ": " + err.Error()
} else if isValidateRun(run) {
if verdict, said = validateRunVerdict(answer); said == "" {
outcome = meshwants.CompositionRunOutcomePassed
}
} else if verdict, said = runVerdict(node, answer); said == "" {
outcome = meshwants.CompositionRunOutcomePassed
}
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
return err
}
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
// stands then; a key gone could not run.
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
out := map[string]meshwants.CompositionRun{}
for {
wake := r.wake()
open := false
for _, key := range keys {
run, _, ok, err := r.read(key)
switch {
case err != nil:
open = true
continue
case !ok:
at := r.now().UTC()
candidate, machine, _ := strings.Cut(key, ".")
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
case run.Ended == nil:
open = true
}
out[key] = run
}
if !open {
return out
}
select {
case <-ctx.Done():
return out
case <-wake:
}
}
}
// unescape is a key's part as a name again; a part that does not read stays as it is.
func unescape(part string) string {
if name, err := keys.UnescapeName(part); err == nil {
return name
}
return part
}
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
out := []meshwants.Shared{}
for _, key := range keys {
run, ok := runs[key]
if !ok {
if run, _, ok, _ = r.read(key); !ok {
continue
}
}
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
}
return out
}
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var ids []string
var errs []error
for _, key := range all {
if !isRunKey(key) {
continue
}
var left []string
var leaving []meshwants.CompositionReader
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
left, leaving = nil, nil
keep := c.Readers[:0:0]
for _, reader := range c.Readers {
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
left = append(left, reader.ID)
leaving = append(leaving, reader)
continue
}
keep = append(keep, reader)
}
c.Readers = keep
return len(left) > 0
})
if err != nil {
errs = append(errs, err)
continue
}
if !ok || len(left) == 0 {
continue
}
ids = append(ids, left...)
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
r.sayWaits(run, leaving, "superseded")
}
if stop != nil {
stop(left) // the steps that left say no end, the run's included
}
if run.Ended != nil || len(run.Readers) > 0 {
continue
}
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
if r.drop != nil {
if err := r.drop(run.Ask, why); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
errs = append(errs, err)
}
}
return ids, errors.Join(errs...)
}
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
func passedOn(module string, run meshwants.CompositionRun) bool {
e := judgeVerdict(module, run) // the unlisted modules change only the words, never whether it passed
return e.failed == "" && e.couldNot == ""
}
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
})
}
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
// not (A-470d).
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
key := runKeyOf(candidate, machine)
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if !ok {
return r.join(reader, candidate, machine, ask)
}
at := formerOf(run.Readers, reader.Pipeline)
ended := run.Ended != nil
keep := ended && at >= 0 && passedOn(reader.Module, run)
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
if keep || read {
if at >= 0 && run.Readers[at].ID == reader.ID {
return joined{key: key, run: run}, nil
}
var former []meshwants.CompositionReader
if at >= 0 {
former = []meshwants.CompositionReader{run.Readers[at]}
run.Readers[at] = reader
} else {
run.Readers = append(run.Readers, reader)
}
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
for i, c := range run.Readers {
if i == at {
continue
}
c.Rerun, c.By = !passedOn(c.Module, run), ""
if c.Rerun {
c.By = reader.By
}
fresh.Readers = append(fresh.Readers, c)
}
fresh.Readers = append(fresh.Readers, reader)
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
}
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// write writes a run's value at its revision.
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
raw, err := json.Marshal(run)
if err != nil {
return err
}
if _, err := r.kv.update(key, raw, rev); err != nil {
return err
}
r.heard()
return nil
}
@@ -1,404 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-assigner/internal/buckets"
)
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
// pass and moves b; a superseded step leaves
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
b := startBed(t, "")
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const commit = "1111111111111111111111111111111111111111"
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
for _, m := range []string{"a", "b", "c"} {
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2"} {
for _, m := range []string{"a", "b", "c"} {
b.place(t, node, m)
}
}
var mu sync.Mutex
inputsAsked := map[string]int{} // by candidate and node
var drops []string
respond := func(subject string, f func(m *nats.Msg) any) {
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
node := strings.Split(m.Subject, ".")[5]
var in nodeengine.ComposeCandidateInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
inputsAsked[in.Candidate+" "+node]++
mu.Unlock()
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
})
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
var in nodebuildgo.DropInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
drops = append(drops, in.ID)
mu.Unlock()
return map[string]any{}
})
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
// repeated message id is never heard (A-470q).
ended := make(chan meshwants.StepEnded, 32)
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
t.Fatal(err)
}
heard, err := runner.Consume(func(m jetstream.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data(), &e)
_ = m.Ack()
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(heard.Stop)
subscribed(t, nc)
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
queue, _ := buildstep.AskStream(buildstep.AskGo)
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
step := func(id, module, run string, rerun bool) {
t.Helper()
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
if run != "" {
s.Pipeline = "run-" + run + "." + module
}
if rerun {
s.By = "op@n1"
}
raw, _ := json.Marshal(s)
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = raw
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
}
type asked struct {
ask buildstep.StepAsk
machine string
}
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
asks := func(n int) []asked {
t.Helper()
var out []asked
for range n {
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
if err != nil {
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
out = append(out, asked{a, req.Machines[0]})
}
return out
}
noMoreAsks := func(what string) {
t.Helper()
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
}
}
say := func(event string, node string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
t.Fatal(err)
}
}
answer := func(a asked, machine string) {
t.Helper()
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
}
ends := func(n int, what string) map[string]meshwants.StepEnded {
t.Helper()
out := map[string]meshwants.StepEnded{}
for len(out) < n {
select {
case e := <-ended:
out[e.ID] = e
case <-time.After(20 * time.Second):
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
}
}
return out
}
runKeys := func(candidate string) []string {
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
var out []string
for _, k := range all {
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
out = append(out, k)
}
}
return out
}
// readers waits, through a watch of the key, until the run of key has n readers.
readers := func(key string, n int) {
t.Helper()
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
select {
case e := <-w.Updates():
var run meshwants.CompositionRun
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
return
}
case <-deadline:
t.Fatalf("the run %s has not %d readers", key, n)
}
}
}
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
composes := `{"composes":true,"blamed":[]}`
// One candidate, three steps, two machines: two runs.
for _, m := range []string{"a", "b", "c"} {
step("step-"+m, m, "", false)
}
first := asks(2)
noMoreAsks("three steps of one candidate")
for _, a := range first {
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
}
got := ends(3, "the three steps")
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
t.Fatalf("the steps of one candidate: %+v", got)
}
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
}
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
step("step-r1", "a", "r1", false)
for _, a := range asks(2) {
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
}
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
t.Fatalf("a step of another candidate: %+v", e)
}
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
from = b.journal.mark()
step("step-r2", "a", "r2", false)
bound := asks(2)
var n1 asked
for _, a := range bound {
if a.machine == "n1" {
n1 = a
}
}
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
for i := range 3 { // past the 2 s bound, each beat within it
select {
case e := <-ended:
t.Fatalf("a run saying its progress ended: %+v", e)
case <-time.After(time.Second):
}
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
runs := b.buckets[buckets.CompositionRuns()]
all, _ := runs.Keys(b.ctx)
held := ""
for _, k := range all {
if strings.HasPrefix(k, runBoundPrefix) {
if e, err := runs.Get(b.ctx, k); err == nil {
var v boundValue
_ = json.Unmarshal(e.Value(), &v)
held += v.Ask
}
}
}
if held != n1.ask.ID {
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
}
}
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
}
e := ends(1, "the bounds")["step-r2"]
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
t.Fatalf("past the bounds: %+v", e)
}
mu.Lock()
dropped := len(drops)
mu.Unlock()
if dropped != 2 {
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
}
answer(n1, composes) // heard after the bound: ignored and journaled
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
step("step-a2", "a", "", true)
again := asks(1)
noMoreAsks("a rerun of a")
if again[0].machine != "n2" {
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
}
answer(again[0], composes)
moved := ends(2, "the rerun")
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
}
// A superseded step leaves the runs; its other reader goes on.
step("step-r3a", "a", "r3", false)
step("step-r3b", "b", "r3", false)
sup := asks(2)
readers(runKeyOf("run-r3", "n1"), 2)
readers(runKeyOf("run-r3", "n2"), 2)
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Branch: trunkBranch, Source: 1, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Movers: []mpr.MoverEnded{{Module: "a", Steps: []mpr.StepResult{}}}})
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
t.Fatal(err)
}
for _, a := range sup {
answer(a, composes)
}
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the other reader: %+v", got)
}
select {
case e := <-ended:
t.Fatalf("a superseded step said its end: %+v", e)
case <-time.After(200 * time.Millisecond):
}
// Restarted during a run: the new assigner takes it up by its ask.
step("step-r4", "c", "r4", false)
restart := asks(2)
stop()
for _, a := range restart {
answer(a, composes)
}
from = b.journal.mark()
b.startStoppable(t)
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
t.Fatalf("the step taken up: %+v", e)
}
// compositions-drop: the readers ended naming the drop, the key deleted.
step("step-r5", "a", "r5", false)
asks(2)
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
}
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
t.Fatalf("a reader of the dropped run: %+v", e)
}
if keys := runKeys("run-r5"); len(keys) != 1 {
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
}
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
for i := range 2 {
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
t.Fatal(err)
}
}
step("step-r6", "a", "r6", false)
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
t.Fatalf("at the cap: %+v", e)
}
}
@@ -1,642 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
type runsMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
ttl map[string]time.Duration
bytes int64
n uint64
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
beforeDel func()
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
beforeArm func()
}
func (k *runsMem) another() {
k.mu.Lock()
f := k.beforeDel
k.beforeDel = nil
k.mu.Unlock()
if f != nil {
f()
}
}
func newRunsMem() *runsMem {
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
}
func (k *runsMem) get(key string) ([]byte, uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
return k.vals[key], k.revs[key], nil
}
func (k *runsMem) create(key string, v []byte) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) delAt(key string, rev uint64) error {
k.another()
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; !ok {
return nil
}
if k.revs[key] != rev {
return errRunChanged
}
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
return nil
}
func (k *runsMem) keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
slices.Sort(out)
return out, nil
}
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
k.mu.Lock()
f := k.beforeArm
k.beforeArm = nil
k.mu.Unlock()
if f != nil {
f()
}
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
k.mu.Lock()
defer k.mu.Unlock()
k.n++
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
return k.n, nil
}
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
func (k *runsMem) expire(key string) {
k.mu.Lock()
defer k.mu.Unlock()
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
}
func (k *runsMem) used() (int64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.bytes > 0 {
return k.bytes, nil
}
var n int64
for _, v := range k.vals {
n += int64(len(v))
}
return n, nil
}
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
step := func(id, module string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
}
t.Run("key", func(t *testing.T) {
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
t.Errorf("the key is %q, want each part escaped as a key's name", k)
}
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
t.Error("a bound key is taken for a run's or a run's for a bound key")
}
})
t.Run("one run for two steps at once", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var wg sync.WaitGroup
got := make([]joined, 2)
for i, m := range []string{"a", "b"} {
wg.Add(1)
go func() {
defer wg.Done()
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
if err != nil {
t.Error(err)
}
got[i] = j
}()
}
wg.Wait()
runKeys := func() []string {
all, _ := kv.keys()
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
}
if got[0].started == got[1].started {
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
}
keys := runKeys()
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
}
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
}
})
t.Run("could not run is never read", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
return true
})
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
}
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
return true
})
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
t.Errorf("a passed run is not read: %+v", j)
}
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
t.Errorf("a step of another candidate reads the first's run: %+v", j)
}
})
t.Run("cap", func(t *testing.T) {
kv := newRunsMem()
kv.bytes = 16 << 20
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
t.Errorf("at the cap: %v, want the fill named", err)
}
})
t.Run("kept", func(t *testing.T) {
kv := newRunsMem()
at := now
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
for _, c := range []string{"main.1", "main.2"} {
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
t.Fatal(err)
}
}
old := now.Add(-25 * time.Hour)
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
recent := now.Add(-time.Hour)
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
t.Fatal(err)
}
keys, _ := kv.keys()
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
}
}
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
}
})
t.Run("a listing that misses a key being written", func(t *testing.T) {
kv := &missingRuns{runsMem: newRunsMem()}
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.hide = runKeyOf("main.3", "n1")
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
t.Fatalf("the stand's listing does not hide the key")
}
r.saw(kv.hide, true) // as the state's watch hands it
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
}
r.saw(kv.hide, false)
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
}
})
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
// replacing the key, and its untaken bound, stand.
for _, c := range []struct {
name string
ended time.Time
act func(r *compositionRuns, key string) error
}{
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
return err
}},
} {
t.Run(c.name, func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
if !c.ended.IsZero() {
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
}
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
for {
raw, rev, _ := kv.get(key)
var run meshwants.CompositionRun
_ = json.Unmarshal(raw, &run)
if run.Ended == nil {
at := now
run.Ended, run.Outcome = &at, "failed"
}
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
if _, err := kv.update(key, fresh, rev); err == nil {
break
}
}
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
if err := c.act(r, key); err != nil {
t.Fatal(err)
}
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
}
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
t.Error("the untaken bound the rerun's run armed is deleted")
}
})
}
t.Run("settings", func(t *testing.T) {
dir := t.TempDir()
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
env := map[string]string{}
for name, content := range files {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
env[name] = p
}
s, err := runSettingsOf(func(k string) string { return env[k] })
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
t.Errorf("settings %+v %v", s, err)
}
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
t.Errorf("a bound of no unit: %v", err)
}
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
t.Errorf("no files: %+v %v, want the defaults", s, err)
}
})
raw, _ := json.Marshal(step("x", "a"))
if !strings.Contains(string(raw), `"id":"x"`) {
t.Errorf("a reader names no ask id: %s", raw)
}
}
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
// are taken up at the start.
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
reader := func(id string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
}
type said struct{ event, how, module string }
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
kv := newRunsMem()
var saids []said
var drops, journal []string
var mu sync.Mutex
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
mu.Lock()
journal = append(journal, fmt.Sprintf(f, a...))
mu.Unlock()
})
r.say = func(event string, body any, _ string) error {
w := body.(meshwants.Wait)
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
t.Errorf("the wait said %+v", w)
}
saids = append(saids, said{event, w.How, w.Module})
return nil
}
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
return kv, r, &saids, &drops, &journal
}
key := runKeyOf("main.3", "n1")
t.Run("taken, progress, bound", func(t *testing.T) {
kv, r, saids, drops, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
_, untaken, _ := kv.get(runUntakenPrefix + key)
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
}
before := kv.revs[runBoundPrefix+key]
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
t.Fatalf("a step-progress does not arm the bound again: %v", err)
}
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
}
if got := judgeRun("a", run); got.couldNot != run.Said {
t.Errorf("a reader of the run past its bound: %+v", got)
}
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Fatal(err)
}
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
}) {
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
}
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
t.Errorf("the bound keys stay past the run's end: %v", keys)
}
})
t.Run("untaken", func(t *testing.T) {
kv, r, saids, drops, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
}
r.expired(runUntakenPrefix + key) // a run ended takes nothing
if len(*drops) != 1 {
t.Errorf("an ended run is dropped again: %v", *drops)
}
})
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
return func() {
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
kv.expire(runBoundPrefix + key)
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
}
taken := func(t *testing.T, r *compositionRuns) {
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
act func(t *testing.T, kv *runsMem, r *compositionRuns)
ended bool
bound string // the ask the bound key holds, "" for none
untaken string
journal string
}{
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
kv.beforeArm = rerun(kv, r)
taken(t, r)
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
kv.beforeArm = rerun(kv, r)
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
r.expired(runBoundPrefix + key)
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
rerun(kv, r)()
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
r.expired(runBoundPrefix + key)
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
}
}, false, "", "run-1", ""},
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
}, false, "run-1", "", ""},
} {
t.Run(c.name, func(t *testing.T) {
kv, r, _, _, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, kv, r)
run, _, _, _ := r.read(key)
if (run.Ended != nil) != c.ended {
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
}
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
if held, _, _, _ := r.boundOf(prefix + key); held != want {
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
}
}
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
t.Errorf("the journal %v does not say %q", *journal, c.journal)
}
})
}
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
// superseded.
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
for _, c := range []struct {
name string
act func(t *testing.T, r *compositionRuns)
want []said
}{
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
again := reader("b")
again.Rerun = true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b")}},
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
again := reader("a2")
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), gone("a"), held("a2")}},
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if _, err := r.leave("main.3.a", nil); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b"), gone("a")}},
} {
t.Run(c.name, func(t *testing.T) {
_, r, saids, _, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, r)
if !slices.Equal(*saids, c.want) {
t.Errorf("said %v, want %v", *saids, c.want)
}
})
}
t.Run("taken up at the start", func(t *testing.T) {
_, r, _, _, _ := setup(t)
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
t.Fatal(err)
}
}
ended := now
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"c", "d"} {
c := reader(id)
c.Ended, c.Rerun = &ended, id == "d"
run.Readers = append(run.Readers, c)
}
return true
})
readers, err := r.readersOfOpen()
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
t.Fatalf("the readers taken up: %+v %v", readers, err)
}
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
t.Errorf("the runs of reader a: %v", keys)
}
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
// `<id>.<the open run's ask>`.
ids := make(chan [2]string, 3)
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
},
say: func(_ string, body any, id string) error {
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
return nil
}}
steps.takeUp(context.Background(), r)
got := map[string]string{}
for range 3 {
select {
case e := <-ids:
got[e[0]] = e[1]
case <-time.After(5 * time.Second):
t.Fatalf("the ends of the take-up: %v", got)
}
}
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
}
})
}
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
type missingRuns struct {
*runsMem
hide string
}
func (k *missingRuns) keys() ([]string, error) {
all, err := k.runsMem.keys()
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
}
@@ -1,162 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
// The tools that call the two verbs.
const (
compositionsTool = "assigner_compositions"
compositionsDropTool = "assigner_compositions_drop"
)
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
// machine named only those, and the state's fill against its cap.
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
all, err := r.keys()
if err != nil {
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
}
slices.Sort(all)
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
continue
}
run.Key = key
out.Runs = append(out.Runs, run)
}
out.Fill, err = r.fill()
return out, err
}
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
// the readers it ended.
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
why := strings.TrimSpace(in.Why)
if in.Candidate == "" || in.Machine == "" || why == "" {
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
}
key := runKeyOf(in.Candidate, in.Machine)
run, _, ok, err := r.read(key)
if err != nil {
return meshwants.CompositionsDropAnswer{}, err
}
if !ok {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
}
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
answer := meshwants.CompositionsDropAnswer{Run: run}
answer.Run.Key = key
if run.Ended == nil {
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
ids := make([]string, 0, len(run.Readers))
for _, reader := range run.Readers {
ids = append(ids, reader.ID)
}
c.leave(ids) // their waits say no end of their own
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
return answer, err
}
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
for _, reader := range run.Readers {
if reader.Ended != nil && !reader.Rerun {
continue // it ended passed already and keeps its verdict (A-470r)
}
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
if reader.Ended != nil {
again = run.Ask
}
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
answer.Ended = append(answer.Ended, reader)
}
}
// The dropped run's key goes while it holds that run: a run started on it since stands.
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
r.heard()
return answer, err
}
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsInput
if len(raw) > 0 {
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
}
return r.compositions(in)
},
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsDropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.drop(r, record, caller, in)
},
}
}
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
call := func(verb string, args map[string]any, names ...string) (any, error) {
in := map[string]any{}
for _, n := range names {
if v := arg(args, n); v != "" {
in[n] = v
}
}
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
if err != nil {
return nil, err
}
var out any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
}
return out, nil
}
return []stdio.Tool{
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositions, args, "candidate", "machine")
}},
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
}},
}
}
@@ -1,91 +0,0 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
ends := newSaidEvents()
var endIDs []string // the message id of each end said
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
endIDs = append(endIDs, id)
return ends.say(event, body, id)
}}
moves := 0
move := func(meshwants.CompositionRun) { moves++ }
r.moved.Store(&move)
for _, m := range []string{"n1", "n2"} {
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
t.Fatal(err)
}
}
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"ask-p", "ask-q"} {
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
c.Ended, c.Rerun = &now, id == "ask-q"
run.Readers = append(run.Readers, c)
}
return true
})
var recorded []meshhandactsclient.Request
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
verbs := compositionVerbs(r, c, record)
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
read, _ := out.(meshwants.CompositionsAnswer)
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
t.Fatalf("compositions = %+v %v", out, err)
}
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
}
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
dropped, _ := out.(meshwants.CompositionsDropAnswer)
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
}
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
t.Errorf("the reader's end %+v", end)
}
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
}
// The tools call the verbs with what they were given.
var asked []string
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
})
for _, tool := range tools {
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
t.Fatal(err)
}
}
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
t.Errorf("the tools asked %v", asked)
}
}
@@ -1,252 +0,0 @@
package main
import (
"context"
"errors"
"sync"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
// composition step of one module's pipeline that the runner asks, runs it, and says its end as the seat's event
// step-ended.<node>, the ask's id as the event's id and the event's `id`, which the runner holds the end to.
// stepOutcome is what one composition step came to.
type stepOutcome struct {
Outcome string
Said string
Warnings []string
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
Shared []meshwants.Shared
}
// compositionSteps takes the composition steps and says their ends.
type compositionSteps struct {
node string
// say says one event of the seat mesh-wants (wantsPublisher).
say publisher
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
// validate runs one validate step and resumeValidate judges one taken up at the start (validatestep.go, design 47 H §745);
// nil takes no validate step.
validate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
resumeValidate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
// writes none.
ended func(id string, run meshwants.StepRun, at time.Time)
now func() time.Time
logf func(string, ...any)
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
// taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not
// pulled yet are kept there too (H-A-404).
taken takenStore
takenBound time.Duration
// inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777).
inProgress func() time.Duration
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is
// dropped (the cause an *askDrop).
stops map[string]context.CancelCauseFunc
// held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said.
held map[string][]queueMsg
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
// composition runs it waits on (§723, §729).
type stepPhase struct {
mu sync.Mutex
phase string
shared []meshwants.Shared
}
// setShared names the runs the step ctx runs waits on, when it runs one.
func setShared(ctx context.Context, shared []meshwants.Shared) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.shared = shared
p.mu.Unlock()
}
}
func (p *stepPhase) waitsOn() []meshwants.Shared {
p.mu.Lock()
defer p.mu.Unlock()
return p.shared
}
type phaseKey struct{}
// setPhase sets the phase of the step ctx runs, when it runs one.
func setPhase(ctx context.Context, phase string) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.phase = phase
p.mu.Unlock()
}
}
func (p *stepPhase) get() string {
p.mu.Lock()
defer p.mu.Unlock()
return p.phase
}
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
switch {
case step == meshwants.StepRunStepComposition && resumed:
return c.resume
case step == meshwants.StepRunStepComposition:
return c.run
case step == meshwants.StepRunStepValidate && resumed:
return c.resumeValidate
case step == meshwants.StepRunStepValidate:
return c.validate
}
return nil
}
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
// says nothing: the runner ended it already.
func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
started := c.now().UTC()
ctx, stopWait := context.WithCancelCause(parent)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelCauseFunc{}
}
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
c.mu.Unlock()
stopWait(nil)
return
}
c.stops[id] = stopWait
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait(nil)
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, run.Asker, phase)
stopInProgress := c.inProgressEvery(id)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stopInProgress()
stop()
if ctx.Err() != nil {
if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start
return
}
var d *askDrop
if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401)
c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again)
return
}
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
c.settle(id, true) // the runner ended it already: its ask is over
return
}
c.sayEnd(id, run, &started, o, again)
}
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
ended := c.now().UTC()
if started == nil {
started = &ended
}
step := meshwants.StepEndedStepComposition
if run.Step == meshwants.StepRunStepValidate {
step = meshwants.StepEndedStepValidate
}
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: step, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
msgID := id
if again != "" {
msgID = id + "." + again
}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err)
c.settle(id, false)
return
}
c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777)
// One journal line per ask answered, in the one place it is answered (design 47 H2 §792).
c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome)
if c.ended != nil {
c.ended(id, run, ended)
}
}
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
return func(run meshwants.CompositionRun) {
for _, reader := range run.Readers {
if !reader.Rerun || c.running(reader.ID) {
continue
}
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
}
}
}
// running is whether the step asked under id runs here.
func (c *compositionSteps) running(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.stops[id] != nil
}
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
func (c *compositionSteps) leave(ids []string) {
c.mu.Lock()
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop(nil)
}
}
}
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) {
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(progressEvery)
done := make(chan struct{})
ended := make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
}
}
}()
return func() { close(done); <-ended }
}
@@ -1,398 +0,0 @@
//go:build integration
package main
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §354, §22, §63, §64, §389, A-240, A-252, A-254: the assigner started whole takes the runner's composition step of a
// module placed on three machines, one of them away: it asks the two others' node-engines for their inputs, puts one run ask per
// machine on the Go build seat's check queue under its name, its program the node-engine build each machine runs; the stand-in
// build seat reads each request by the seat mesh-wants' verb input in chunks and says its step-ended with the composer's verdict;
// the step's step-ended fails naming the machine that does not compose, warns of the machine away, and no input stays kept.
func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) {
b := startBed(t, "")
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const engineCommit = "1111111111111111111111111111111111111111"
// The assigner's durable on EVENTS hears, beside the bed's, the build seat's step-ended and the watchman's machine-away and
// machine-back, as the grants part derives them from its consumes; what each node runs it hears on its kept stream (the bed's).
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, meshwatchdogs.SubjectMachineAwayAll, meshwatchdogs.SubjectMachineBackAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
away, _ := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: "n3", At: time.Now().UTC(), LastHeard: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, meshwatchdogs.SubjectMachineAway("n3"), away); err != nil {
t.Fatal(err)
}
// The trunk's catalog half holds the module and the node-engine, the module claiming the seat node-engine.
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: engineCommit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine,
"m": {Module: "m", Fp: "src2:m1", Repository: "novox/m", Commit: engineCommit, Path: ".", Manifest: map[string]any{"module": "m", "version": "0.1"}}}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2", "n3"} {
b.place(t, node, "m")
}
// The node-engines of n1 and n2 answer compose-candidate with their inputs; n3 is never asked.
asked := make(chan string, 8)
engines, err := nc.Subscribe("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) {
node := strings.Split(m.Subject, ".")[5]
asked <- node
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"candidate": "main.3", "node": node, "verdict": "composed", "input": map[string]any{"node": node},
"modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}, "node": node})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engines.Unsubscribe() })
// mesh-catalog's entry answers the node-engine build's entry by its fingerprint.
entries, err := nc.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: engine}, "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = entries.Unsubscribe() })
subscribed(t, nc)
// The runner hears the step's end.
ended := make(chan meshwants.StepEnded, 1)
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data, &e)
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
select {
case <-b.seatListRead: // the seat server admits a caller of input once the seat list is read (A-278)
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
// its end with the composer's verdict: n1 composes, n2 does not.
queue, _ := buildstep.AskStream(buildstep.AskGo)
// A-163: the assigner asks under its seat's own name.
subject := "mesh.seat.node-build-go.accept.check.mesh-wants"
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
programs := map[string]string{}
for range 2 {
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
if err != nil {
t.Fatalf("no run ask is put: %v", err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
if len(req.Seats) == 0 { // the revision names no definition: the request carries the trunk seat list's (A-467)
t.Errorf("the request of %s carries no seat definition", a.ID)
}
node := req.Machines[0]
programs[node] = a.Run.Program.Commit
m := composer.Machine{Composes: true}
if node == "n2" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(v), At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
t.Fatal(err)
}
}
var e meshwants.StepEnded
select {
case e = <-ended:
case <-b.ctx.Done():
t.Fatal("the composition step said no end")
}
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
e.Warnings[1] != "n2: the composer names no culprits" {
t.Fatalf("the step ended %+v", e)
}
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
t.Errorf("the composers were built at %v, want the node-engine build each machine runs", programs)
}
close(asked)
for node := range asked {
if node == "n3" {
t.Error("the machine away was asked for its input")
}
}
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
inputs := b.buckets[buckets.Inputs()]
w, err := inputs.WatchAll(b.ctx)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
keys, _ := inputs.Keys(b.ctx)
if len(keys) == 0 {
break
}
select {
case <-w.Updates():
case <-deadline:
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
}
}
queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777)
}
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
// chunks on the call's subjects, each acknowledged.
func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
t.Helper()
call := "standin" + strings.Repeat("0", 16)
chunks := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: call}
sub, err := nc.SubscribeSync(chunks.Chunks())
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
body, _ := json.Marshal(meshwants.InputInput{Digest: in.Digest, Call: call})
// On the call's subject naming its caller, the stand-in build seat's account: the seat server admits no unproven caller (A-179).
reply, err := nc.Request("mesh.seat.mesh-wants.call.input.n1~mesh-build-go", body, 10*time.Second)
if err != nil {
t.Fatal(err)
}
raw, said, ok := buscalls.Unwrap(reply.Data)
var a meshwants.InputAnswer
if !ok || json.Unmarshal(raw, &a) != nil || a.Chunks < 1 {
t.Fatalf("the verb input answered %s: %s", reply.Data, said)
}
var out bytes.Buffer
next := func() (string, []byte, error) {
m, err := sub.NextMsg(buscalls.SeatAckWithin)
if err != nil {
return "", nil, err
}
return m.Subject, m.Data, nil
}
ack := func(subject string) error { _, err := nc.Request(subject, nil, 10*time.Second); return err }
if _, err := chunks.Receive(a.Chunks, next, ack, &out); err != nil {
t.Fatal(err)
}
if err := in.Verify(out.Bytes()); err != nil {
t.Fatal(err)
}
return out.Bytes()
}
// Design 47 H §745 (acceptance test 117, the assigner's half; H-A-108), started whole on the bus: the assigner serves the step
// validate on the seat mesh-wants. A take-in whose manifest names an unknown field fails it `manifest-refused` naming it, the
// validator being the revision's own entry of the module holding the seat mesh-catalog, run on the Go build seat over the
// revision's manifests and the previous revision's; a take-in that changes a manifest with no problem passes it; a take-in whose
// entries make a cycle fails it `cycle` naming the cycle, with no validator asked.
func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T) {
b := startBed(t, "")
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const commit = "2222222222222222222222222222222222222222"
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commit, Path: "modules/mesh-delivery-indexer",
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
entry := func(module, digest string, manifest map[string]any, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
manifest["module"] = module
return meshcatalog.CatalogEntry{Module: module, Manifest: manifest, ManifestDigest: digest, Edges: edges}
}
b.cat.mu.Lock()
b.cat.rev[trunkBranch] = 9
b.cat.at = map[string]map[int64][]meshcatalog.CatalogEntry{trunkBranch: {
6: {indexer, entry("web", "d-web-1", map[string]any{"version": "1"})},
7: {indexer, entry("web", "d-web-2", map[string]any{"version": "2", "colour": "red"})},
8: {indexer, entry("web", "d-web-3", map[string]any{"version": "3"})},
9: {indexer, entry("a", "d-a", map[string]any{}, meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}),
entry("b", "d-b", map[string]any{})},
}}
b.cat.mu.Unlock()
ended := make(chan meshwants.StepEnded, 4)
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data, &e)
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
// Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken)
// and ended after its start.
taken := make(chan *nats.Msg, 8)
takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = takens.Unsubscribe() })
subscribed(t, nc)
putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a",
Branch: trunkBranch, Revision: 9, Commit: commit})
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second)
select {
case m := <-taken:
var at meshwants.AskTakenEvent
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") {
t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err)
}
case <-time.After(30 * time.Second):
t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped")
}
select {
case e := <-ended:
if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
}
case <-time.After(30 * time.Second):
t.Fatal("the validate ask put while the assigner was stopped said no end")
}
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
queue, _ := buildstep.AskStream(buildstep.AskGo)
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
ask := func(revision int64, module string, run func()) meshwants.StepEnded {
t.Helper()
id := fmt.Sprintf("validate-%d", revision)
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
Branch: trunkBranch, Revision: revision, Commit: commit})
var asked meshwants.StepRun
_ = json.Unmarshal(step, &asked)
putStep(b.ctx, t, b.js, id, asked)
if run != nil {
run()
}
for {
select {
case e := <-ended:
if e.ID == id {
return e
}
case <-time.After(30 * time.Second):
t.Fatalf("the validate step of revision %d said no end", revision)
}
}
}
// The stand-in Go build seat runs the validator: it refuses a manifest naming the field colour, and accepts every other.
validator := func() {
t.Helper()
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
if err != nil {
t.Fatalf("no validator run is asked: %v", err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil || a.Run.Program.Repository != "novox/mesh-delivery" || a.Run.Program.Commit != commit {
t.Fatalf("the validator's run ask %s: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase {
t.Fatalf("the validator's request: %+v %v", req, err)
}
// The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443).
if list, digest := seatListBytes(); req.SeatList != digest {
t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest)
} else if got, _ := json.Marshal(req.Seats); string(got) != string(list) {
t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list))
}
v := composer.Verdict{V: composer.Version}
for _, c := range req.Cases {
reads := map[string]composer.ManifestRead{}
for module, m := range c.Manifests {
reads[module] = composer.ManifestRead{Accepted: true}
if strings.Contains(string(m.Raw), "colour") {
reads[module] = composer.ManifestRead{Refused: `unknown field "colour"`}
}
}
v.Cases = append(v.Cases, composer.CaseVerdict{Name: c.Name, Manifests: reads})
}
out, _ := json.Marshal(v)
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(out), At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
t.Fatal(err)
}
}
if e := ask(7, "web", validator); e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(e.Said, reasonManifestRefused+": ") || !strings.Contains(e.Said, `web: unknown field "colour"`) {
t.Errorf("the validate step of a manifest naming an unknown field ended %+v", e)
}
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
}
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
}
queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777)
}
@@ -1,510 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled.
type fakeMsg struct {
data []byte
h nats.Header
mu sync.Mutex
acks, naks, terms, inProgresss int
}
func (m *fakeMsg) Data() []byte { return m.data }
func (m *fakeMsg) Headers() nats.Header { return m.h }
func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil }
func (m *fakeMsg) Ack() error { return m.count(&m.acks) }
func (m *fakeMsg) Nak() error { return m.count(&m.naks) }
func (m *fakeMsg) Term() error { return m.count(&m.terms) }
func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) }
func (m *fakeMsg) settled() (acks, naks, terms, progress int) {
m.mu.Lock()
defer m.mu.Unlock()
return m.acks, m.naks, m.terms, m.inProgresss
}
// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set.
type fakeTaken struct {
mu sync.Mutex
values map[string][]byte
ttls map[string]time.Duration
refuse error
}
func newFakeTaken() *fakeTaken {
return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}}
}
func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.refuse != nil {
return false, f.refuse
}
key := seat + "." + ask
if _, ok := f.values[key]; ok {
return false, nil
}
f.values[key], f.ttls[key] = []byte(`{}`), bound
return true, nil
}
func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) {
raw, err := json.Marshal(value)
f.mu.Lock()
defer f.mu.Unlock()
f.values[key], f.ttls[key] = raw, ttl
return 1, err
}
func (f *fakeTaken) Delete(key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.values, key)
return nil
}
func (f *fakeTaken) Keys() ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.values {
out = append(out, k)
}
return out, nil
}
// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none.
func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg {
if c.taken == nil {
c.taken = newFakeTaken()
}
m := &fakeMsg{data: body, h: nats.Header{}}
if id != "" {
m.h.Set("x-event-id", id)
}
c.pulled(context.Background(), m)
return m
}
// saidEvents keeps what a publisher says.
type saidEvents struct {
mu sync.Mutex
said map[string]any
ids map[string]string
heard chan string
}
func newSaidEvents() *saidEvents {
return &saidEvents{said: map[string]any{}, ids: map[string]string{}, heard: make(chan string, 8)}
}
func (s *saidEvents) say(event string, body any, id string) error {
s.mu.Lock()
s.said[event], s.ids[event] = body, id
s.mu.Unlock()
s.heard <- event
return nil
}
// AS5 (§22, §354): the assigner takes mesh-wants' accepted kind `step` for the composition step and says the seat's
// step-ended.<node> with the ask's id, what the step was asked at and what it came to; an ask of another step is left.
func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
said := newSaidEvents()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
var asked meshwants.StepRun
var lines []string
var linesMu sync.Mutex
logf := func(f string, a ...any) {
linesMu.Lock()
lines = append(lines, fmt.Sprintf(f, a...))
linesMu.Unlock()
}
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if m := pullOne(c, "ask-0", other); m.terms != 1 {
t.Fatal("a step ask of another step is taken")
}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
if m := pullOne(c, "", body); m.terms != 1 {
t.Fatal("a step ask with no id is taken")
}
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777)
t.Fatalf("said %s first, want ask-taken", e)
}
select {
case <-said.heard:
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
event := meshwants.EventStepEnded + ".n1"
end, ok := said.said[event].(meshwants.StepEnded)
if !ok || said.ids[event] != "ask-1" {
t.Fatalf("said %v under %v, want step-ended.n1 under the ask's id", said.said, said.ids)
}
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
t.Fatalf("step-ended %+v", end)
}
// Design 47 H2 §792: the ask answered is one journal line, said where its end is said.
linesMu.Lock()
answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") })
linesMu.Unlock()
if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) {
t.Errorf("the journal says %q, want one answered line", answered)
}
// Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at
// the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop.
t.Run("work queue", queueCases)
}
func queueCases(t *testing.T) {
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4})
type bed struct {
c *compositionSteps
said *saidEvents
taken *fakeTaken
runs chan string
release chan struct{}
tick chan time.Time
}
newBed := func() *bed {
b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)}
b.said.heard = make(chan string, 16)
run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome {
b.runs <- id
select {
case <-b.release:
case <-ctx.Done():
}
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}
b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute,
validate: run, inProgress: func() time.Duration { return 20 * time.Second },
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d == 20*time.Second {
return b.tick, func() {}
}
return make(chan time.Time), func() {}
}}
return b
}
ended := func(t *testing.T, b *bed) meshwants.StepEnded {
t.Helper()
for {
select {
case e := <-b.said.heard:
if e == meshwants.EventStepEnded+".n1" {
b.said.mu.Lock()
defer b.said.mu.Unlock()
return b.said.said[e].(meshwants.StepEnded)
}
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
}
}
settledAt := func(t *testing.T, m *fakeMsg, acks int) {
t.Helper()
deadline := time.After(5 * time.Second)
for {
if a, _, _, _ := m.settled(); a == acks {
return
}
select {
case <-deadline:
t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks)
case <-time.After(time.Millisecond):
}
}
}
t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-1", body)
if id := <-b.runs; id != "ask-1" {
t.Fatalf("ran %s", id)
}
b.said.mu.Lock()
taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent)
takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"]
b.said.mu.Unlock()
if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute {
t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls)
}
b.tick <- at
b.tick <- at
// handed again while it runs: held for the run, which answers both at its one end
again := pullOne(b.c, "ask-1", body)
if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 {
t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p)
}
close(b.release)
if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
settledAt(t, again, 1)
select {
case id := <-b.runs:
t.Fatalf("the ask handed again ran again (%s)", id)
default:
}
})
t.Run("taken before is run again, said no ask-taken", func(t *testing.T) {
b := newBed()
_, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute)
close(b.release)
m := pullOne(b.c, "ask-2", body)
<-b.runs
ended(t, b)
settledAt(t, m, 1)
b.said.mu.Lock()
_, said := b.said.said[meshwants.EventAskTaken+".ask-2"]
b.said.mu.Unlock()
if said {
t.Fatal("ask-taken is said again for an ask taken before")
}
})
t.Run("a refused take is handed again", func(t *testing.T) {
b := newBed()
b.taken.refuse = errors.New("no bus")
m := pullOne(b.c, "ask-3", body)
if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 {
t.Fatalf("nakked %d, ran %d", n, len(b.runs))
}
})
t.Run("dropped while it runs", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-4", body)
<-b.runs
if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil {
t.Fatal("a drop with no why is taken")
}
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"})
if err != nil || a["held"] != true {
t.Fatalf("drop answered %v, %v", a, err)
}
e := ended(t, b)
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
})
t.Run("dropped before it was pulled", func(t *testing.T) {
b := newBed()
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"})
if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute {
t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls)
}
m := pullOne(b.c, "ask-5", body)
e := ended(t, b)
if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
if len(b.runs) != 0 {
t.Fatal("a dropped ask ran")
}
if v, _ := b.taken.Get("dropped.ask-5"); v != nil {
t.Fatal("the drop's key stays once its ask ended")
}
})
t.Run("the asks tools", func(t *testing.T) {
taken := newFakeTaken()
_, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute)
_, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute)
waiting := func(seat string) (stdio.Waiting, error) {
if seat != meshwants.Seat {
t.Errorf("the waiting asks of %s are read", seat)
}
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3},
{ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil
}
var asked []string
ask := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
if strings.HasSuffix(key, "."+meshwants.VerbDrop) {
return json.RawMessage(`{"held":false}`), nil
}
return json.RawMessage(`{}`), nil
}
tools := map[string]stdio.Tool{}
for _, tool := range asksTools(waiting, taken, ask) {
tools[tool.Name] = tool
}
out, err := tools[asksTool].Run(map[string]any{})
a, _ := out.(map[string]any)
w, _ := a["waiting"].([]map[string]any)
d, _ := a["dropped"].([]map[string]any)
if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 ||
d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false {
t.Fatalf("assigner_asks answered %+v, %v", out, err)
}
if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 {
t.Fatalf("a drop with no why: %v, asked %v", err, asked)
}
out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"})
if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") ||
!strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) {
t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked)
}
})
}
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
// asker, the phase the run is in and the time, and says none once it has ended.
func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said := newSaidEvents()
tick := make(chan time.Time)
stopped := make(chan struct{})
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
release := make(chan struct{})
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d != time.Minute {
t.Errorf("the progress ticks every %s, want a minute", d)
}
return tick, func() { close(stopped) }
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
<-said.heard // ask-taken, at the pull (H2 §777)
progress := meshwants.EventStepProgress + ".n1"
for range 2 {
tick <- at
if e := <-said.heard; e != progress {
t.Fatalf("said %s on a tick, want %s", e, progress)
}
}
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
t.Fatalf("step-progress %+v", p)
}
close(release)
if e := <-said.heard; e != meshwants.EventStepEnded+".n1" {
t.Fatalf("said %s at the end, want step-ended", e)
}
select {
case <-stopped:
default:
t.Fatal("the ticker is not stopped when the step has ended")
}
}
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
// reader is left", so a later step starts a new run.
func TestASupersededStepLeavesItsRuns(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
step := func(pipeline string) meshwants.StepRun {
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
}
said := newSaidEvents()
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
if err != nil {
t.Error(err)
}
r.await(ctx, []string{j.key})
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 {
t.Fatal("not taken")
}
}
key := runKeyOf("main.3", "n1")
for {
wake := r.wake()
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
break
}
<-wake
}
ids, err := r.leave("main.3.a", c.leave)
run, _, _, _ := r.read(key)
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
}
_, err = r.leave("main.3.b", c.leave)
run, _, _, _ = r.read(key)
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
}
deadline := time.After(5 * time.Second)
for {
c.mu.Lock()
n := len(c.stops)
c.mu.Unlock()
if n == 0 {
break
}
select {
case <-deadline:
t.Fatal("the steps that left still wait")
case <-time.After(time.Millisecond):
}
}
said.mu.Lock()
defer said.mu.Unlock()
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
t.Errorf("a step that left says its end: %v", said.said)
}
}
@@ -1,153 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// engineWords is one node's key of the assigner's state engine-reads (design 47 H-A-104): the words its node-engine said it
// reads in the last manifest-reads-changed the assigner heard of it, and when the engine said them.
type engineWords struct {
Words []string `json:"words"`
At time.Time `json:"at"`
}
// engineReadsKept is what the assigner holds of its state engine-reads (design 47 H-A-104, §772): written when it hears the
// node-engine seat's manifest-reads-changed, and read back from the state's watch at the start, so a restart loses no node's
// words; no verb is asked and no node's record is read in its place.
type engineReadsKept struct {
mu sync.Mutex
by map[string]engineWords
// put writes one node's key; nil keeps the words in memory only (the state could not be opened).
put func(node string, value []byte) error
// changed wakes a pass when a node's words changed; nil wakes nothing.
changed func()
}
// heard keeps what one manifest-reads-changed says of its node: written into the state first, then held.
func (k *engineReadsKept) heard(e nodeengine.ManifestReadsChangedEvent) error {
at, err := time.Parse(time.RFC3339Nano, e.At)
if err != nil {
return fmt.Errorf("at %q is no RFC 3339 time", e.At)
}
v := engineWords{Words: slices.Clone(e.ManifestReads), At: at}
if k.put != nil {
raw, err := json.Marshal(v)
if err != nil {
return err
}
if err := k.put(e.Node, raw); err != nil {
return fmt.Errorf("the words of %s are not written into the state engine-reads: %w", e.Node, err)
}
}
k.hold(e.Node, v, false)
return nil
}
// take takes in one key of the state as its watch hands it: at the start every node's words, later the assigner's own writes
// (which change nothing) and a key dropped.
func (k *engineReadsKept) take(node string, raw []byte, deleted bool) error {
if deleted {
k.hold(node, engineWords{}, true)
return nil
}
var v engineWords
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the key %s of the state engine-reads does not read: %w", node, err)
}
k.hold(node, v, false)
return nil
}
// hold sets one node's words, waking a pass when they changed.
func (k *engineReadsKept) hold(node string, v engineWords, deleted bool) {
k.mu.Lock()
old, had := k.by[node]
switch {
case deleted:
delete(k.by, node)
default:
if k.by == nil {
k.by = map[string]engineWords{}
}
k.by[node] = v
}
changed := (had && deleted) || (!deleted && (!had || !old.At.Equal(v.At) || !slices.Equal(old.Words, v.Words)))
k.mu.Unlock()
if changed && k.changed != nil {
k.changed()
}
}
// words is what node's engine said it reads, and whether the assigner holds its key; nil k holds none.
func (k *engineReadsKept) words(node string) ([]string, bool) {
if k == nil {
return nil, false
}
k.mu.Lock()
defer k.mu.Unlock()
v, ok := k.by[node]
return slices.Clone(v.Words), ok
}
// nodes is every node whose key the assigner holds, sorted.
func (k *engineReadsKept) nodes() []string {
if k == nil {
return nil
}
k.mu.Lock()
defer k.mu.Unlock()
return slices.Sorted(maps.Keys(k.by))
}
// engineReadsTool lists the assigner's state engine-reads: what each node's node-engine said it reads, and when.
const engineReadsTool = "assigner_engine_reads"
// readEngineReads is the state engine-reads, every node's key or node's alone.
func readEngineReads(st store, node string) (any, error) {
keys := []string{node}
if node == "" {
var err error
if keys, err = st.keys(engineReadsState); err != nil {
return nil, err
}
slices.Sort(keys)
}
out := map[string]engineWords{}
for _, key := range keys {
raw, _, found, err := st.get(engineReadsState, key)
if err != nil {
return nil, err
}
if !found {
if node != "" {
return nil, fmt.Errorf("the assigner holds no words of %s: its node-engine said no %s it heard", node, nodeengine.EventManifestReadsChanged)
}
continue
}
var v engineWords
if err := json.Unmarshal(raw, &v); err != nil {
return nil, errors.Join(fmt.Errorf("the key %s does not read", key), err)
}
out[key] = v
}
return map[string]any{"nodes": out}, nil
}
// engineReadsTools is the tool over the runtime's state.
func engineReadsTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: engineReadsTool, Description: "What each node's node-engine says it reads of a manifest, as the " +
"assigner keeps it in its state engine-reads (design 47 H-A-104): per node the words of the last manifest-reads-changed " +
"it heard and when the engine said them. A node with no key states nothing the assigner heard, so its wants are held " +
"awaiting its hand placement (§772). With node, that node's key alone.",
Input: schema(map[string]string{"node": "one node; every node when left out"}, nil),
Run: func(a map[string]any) (any, error) { return readEngineReads(st, arg(a, "node")) }}}
}
@@ -1,82 +0,0 @@
package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
)
// runtimeObjects is the kept data assignment-revisions as the bundle reaches it through its runtime (the old objects
// section of a foundation part, design 47 lines 1362-1366).
type runtimeObjects struct{ o busstate.Objects }
func (r runtimeObjects) Put(_ context.Context, name string, data []byte) error {
_, err := r.o.Put(name, data, "")
return err
}
func (r runtimeObjects) Get(_ context.Context, name string) ([]byte, error) {
d, err := r.o.Get(name)
if err != nil {
return nil, err
}
if d.Info.Name == "" || d.Info.Deleted {
return nil, assigngraph.ErrNoObject
}
return d.Data, nil
}
func (r runtimeObjects) Names(context.Context) ([]string, error) {
infos, err := r.o.List()
if err != nil {
return nil, err
}
names := []string{}
for _, i := range infos {
if !i.Deleted {
names = append(names, i.Name)
}
}
return names, nil
}
// graphVerbs are the seat mesh-wants' verbs revision, entry and difference (design 47 lines 1365-1366: an assignment graph
// revision is served by the seat's verbs), each reading only the kept revisions.
func graphVerbs(o assigngraph.Objects) map[string]func(map[string]any) (any, error) {
return map[string]func(map[string]any) (any, error){
meshwants.VerbRevision: func(a map[string]any) (any, error) {
var in meshwants.RevisionInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Revision(context.Background(), o, in)
},
meshwants.VerbEntry: func(a map[string]any) (any, error) {
var in meshwants.EntryInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Entry(context.Background(), o, in)
},
meshwants.VerbDifference: func(a map[string]any) (any, error) {
var in meshwants.DifferenceInput
if err := decodeArgs(a, &in); err != nil {
return nil, err
}
return assigngraph.Difference(context.Background(), o, in)
},
}
}
// decodeArgs reads a verb's arguments into its input type; a caller's own fields beside them are the runtime's.
func decodeArgs(a map[string]any, into any) error {
raw, err := json.Marshal(a)
if err != nil {
return err
}
return json.Unmarshal(raw, into)
}
@@ -1,96 +0,0 @@
package main
import (
"context"
"fmt"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
return fillIn(ctx, g.objects)
}
// passObjects is the store as the pass reads its fill: a handle of its own where one is given, so the pass's read never shares one
// with a keeping that runs beside it (design 47 H-A-261).
func (g *graphKeeper) passObjects() assigngraph.Objects {
if g.reads != nil {
return g.reads
}
return g.objects
}
// fillIn is the fill of the bucket o, when it says one and has a cap.
func fillIn(ctx context.Context, o assigngraph.Objects) (used, capacity int64, ok bool) {
f, is := o.(assigngraph.Filler)
if !is {
return 0, 0, false
}
used, capacity, err := f.Fill(ctx)
return used, capacity, err == nil && capacity > 0
}
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
at := p.clock().UTC()
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
}
}
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
// its cap), so a fill reached while the process was down is said too.
func (p *passer) fillAtStart(ctx context.Context) {
g := p.graph
used, capacity, ok := fillIn(ctx, g.passObjects())
if !ok {
return
}
switch {
case used >= capacity:
g.setFull(true)
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
case used*5 >= capacity*4:
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
}
}
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
// 80 % again. It answers true while the store is full.
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
g := p.graph
used, capacity, ok := fillIn(ctx, g.passObjects())
if !ok {
return false
}
if g.isFull() && used*5 < capacity*4 {
g.setFull(false)
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
}
if used+assigngraph.NewBytes(ctx, g.passObjects(), c) <= capacity {
g.setFull(false)
return false
}
if !g.isFull() {
g.setFull(true)
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
}
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
return true
}
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
func (p *passer) fillingAfter(ctx context.Context, before int64) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
}
}
@@ -1,90 +0,0 @@
package main
import (
"context"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
type filled struct {
objs map[string][]byte
base int64
cap int64
}
func (f *filled) Put(_ context.Context, name string, data []byte) error {
f.objs[name] = data
return nil
}
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := f.objs[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (f *filled) Names(context.Context) ([]string, error) {
out := []string{}
for n := range f.objs {
out = append(out, n)
}
return out, nil
}
func (f *filled) Fill(context.Context) (int64, int64, error) {
used := f.base
for _, d := range f.objs {
used += int64(len(d))
}
return used, f.cap, nil
}
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
ctx := context.Background()
p := newPasser(kvSet{}, time.Now())
var events []string
p.publishWants = func(event string, body any, id string) error {
if _, ok := body.(meshwants.Fill); ok {
events = append(events, event)
}
return nil
}
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
if len(events) != 0 {
t.Fatalf("events %v", events)
}
o.base, o.cap = 8500, 10000 // past 80 %, with room for the next revision, which the pass judges itself (H-A-261)
p.graph.started = false
p.keepGraph(ctx) // a start again: its first pass judges the fill
if len(events) != 1 || events[0] != meshwants.EventFilling {
t.Fatalf("at the start past 80 %%: %v", events)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
o.base, o.cap = 990, 1000
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
t.Fatalf("a revision that does not fit: events %v", events)
}
p.fullBefore(ctx, c)
if n := len(events); n != 2 {
t.Errorf("full is said once: %v", events)
}
// H-A-261: the verdict is the pass's own: with the store full the pass writes no want, though a revision is kept.
if held := p.keepGraph(ctx); held != heldFull {
t.Errorf("a pass with the store full answers %q, want %q", held, heldFull)
}
o.base = 100
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
t.Errorf("room: events %v", events)
}
if held := p.keepGraph(ctx); held != "" {
t.Errorf("a pass with room and a revision kept answers %q, want the wants written", held)
}
}
@@ -1,512 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
// from the catalog halves it follows, the placements, the first branch of each registration and each node's capability reading,
// keeps it in the objects assignment-revisions and says the seat mesh-wants' event assignment-revision; the wants of the pass
// that follows are written from the revision it holds.
type graphKeeper struct {
objects assigngraph.Objects
// reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects.
reads assigngraph.Objects
// halves is every followed branch's catalog graph revision held now.
halves func() map[string]mcatc.Half
// mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259).
mu sync.Mutex
// handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false
// until one is, and again when its keeping failed or found the store full, so the next pass hands it again.
handedRev meshwants.AssignmentRevision
handed bool
// handedOnce says a revision was handed: the first carries the start's removal (H-A-261).
handedOnce bool
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
kept meshwants.AssignmentRevision
source int64
ok bool
said string
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
full, started bool
// revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision.
revisions *revisionRemover
// computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one.
computed assigngraph.Computed
hasComputed bool
// keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow).
keepMu sync.Mutex
// The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at
// once, in the caller), and wake makes a pass once a keeping changed what the pass reads.
need *keepNeed
keeping bool
background func(func())
wake func()
// A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures
// counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)).
retry time.Duration
after func(d time.Duration, f func())
failures int
}
// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline).
const (
keepingRetrySetting = "keeping-retry"
defaultKeepingRetry = 30 * time.Second
kindKeepingFailed = "keeping-failed"
// keepingFailedAt is the failures in a row that raise keeping-failed.
keepingFailedAt = 3
)
// keepingRetry reads the setting keeping-retry (§781).
func keepingRetry() (time.Duration, error) {
d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
// keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged.
type keepNeed struct {
c *assigngraph.Computed
judge bool
}
// keptNow is the revision the wants follow, with its trunk source, and whether one is kept.
func (g *graphKeeper) keptNow() (meshwants.AssignmentRevision, int64, bool) {
g.mu.Lock()
defer g.mu.Unlock()
return g.kept, g.source, g.ok
}
// isFull says the store did not hold the last revision handed over, so no want is written (design 47 G §181).
func (g *graphKeeper) isFull() bool {
g.mu.Lock()
defer g.mu.Unlock()
return g.full
}
func (g *graphKeeper) setFull(full bool) {
g.mu.Lock()
g.full = full
g.mu.Unlock()
}
// heldNoRevision is the pass's words while no assignment graph revision is kept: it writes no want (design 47 H-A-261).
const heldNoRevision = "held: no assignment graph revision is kept yet, so no want is written until the first keeping ends (H-A-261)"
// heldFull is the pass's words while the next revision does not fit its store (design 47 G §181).
const heldFull = "the assignment graph store is full: no want is written (design 47 G §181)"
// keepGraph computes the revision of this pass, judges the store's fill itself, and hands the revision to the keeping when the
// pass's inputs differ from the last handed. The keeping and the removing of revisions run off the pass loop, one at a time, and the
// pass never waits on them: it judges and writes the wants from the graph it holds (design 47 H-A-259). It answers why the pass
// writes no want, or "" when it writes them: none while the next revision does not fit the store (G §181), and none while no
// revision is kept (H-A-261), whose keeping wakes a pass when it ends.
func (p *passer) keepGraph(ctx context.Context) string {
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return ""
}
c := assigngraph.Compute(p.graphInputs())
g.mu.Lock()
g.computed, g.hasComputed = c, true
g.mu.Unlock()
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
p.logf("the assignment graph leaves out %s", u)
}
}
if !g.started { // the first pass judges the fill once (graphfill.go)
g.started = true
p.fillAtStart(ctx)
}
if p.fullBefore(ctx, c) {
g.mu.Lock()
g.handed = false // handed once it fits
g.mu.Unlock()
return heldFull
}
g.mu.Lock()
same := g.handed && assigngraph.Same(g.handedRev, c.Revision)
first := !g.handedOnce
if !same {
g.handedRev, g.handed, g.handedOnce = c.Revision, true, true
}
g.mu.Unlock()
if !same {
// The start's removal of the revisions not current folds into the first keeping: it keeps the first pass's revision, then
// removes (H-A-261).
p.needKeeping(ctx, keepNeed{c: &c, judge: first})
}
if _, _, ok := g.keptNow(); !ok {
return heldNoRevision
}
return ""
}
// needKeeping hands a need to the keeping: one keeping runs at a time, and a need that comes while one runs is folded into the one
// that waits (the newest revision, and a judging asked by either), so it runs once more (design 47 H-A-259).
func (p *passer) needKeeping(ctx context.Context, n keepNeed) {
g := p.graph
g.mu.Lock()
if w := g.need; w != nil {
n.judge = n.judge || w.judge
if n.c == nil {
n.c = w.c
}
}
g.need = &n
if g.keeping {
g.mu.Unlock()
return
}
g.keeping = true
g.mu.Unlock()
run := func() {
for {
g.mu.Lock()
next := g.need
g.need = nil
if next == nil {
g.keeping = false
g.mu.Unlock()
return
}
g.mu.Unlock()
p.keepOne(ctx, *next)
}
}
if g.background != nil {
g.background(run)
return
}
run()
}
// keepOne keeps one computed revision, says assignment-revision for a revision kept now and removes the
// revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads.
func (p *passer) keepOne(ctx context.Context, n keepNeed) {
g := p.graph
g.keepMu.Lock()
defer g.keepMu.Unlock()
start := time.Now()
changed := false
defer func() {
p.logf("the keeping of the assignment graph took %s", time.Since(start).Round(time.Millisecond))
if changed && g.background != nil && g.wake != nil {
g.wake()
}
}()
if n.c != nil {
_, _, had := g.keptNow()
fresh := p.keepRevision(ctx, *n.c)
_, _, has := g.keptNow()
n.judge = n.judge || fresh
changed = fresh || had != has // the first revision kept: the pass writes the wants it held (H-A-261)
}
if n.judge && g.revisions != nil {
g.revisions.judge() // the revision before ceased to be current
}
}
// keepRevision keeps c when the store holds it, and says whether a revision was kept now.
func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool {
fresh, _ := p.keepRevisionOf(ctx, c)
return fresh
}
// keepRevisionOf is keepRevision answering the fault the keeping met too.
func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) {
g := p.graph
again := func() { // the next pass hands this revision again
g.mu.Lock()
g.handed = false
g.mu.Unlock()
}
used, _, _ := g.fillOf(ctx)
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
if err != nil {
again()
p.keepingFailed(ctx, c, err)
return false, err
}
p.keepingSucceeded()
var source int64
for _, cat := range rev.Catalogs {
if cat.Branch == p.w.trunkName() {
source = cat.Source
}
}
g.mu.Lock()
g.kept, g.ok, g.source = rev, true, source
g.mu.Unlock()
if !fresh {
return false, nil
}
p.fillingAfter(ctx, used)
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
}
return true, nil
}
// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed
// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed
// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls.
func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) {
g := p.graph
if strings.TrimSpace(in.Why) == "" {
return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now")
}
if g == nil || g.objects == nil {
return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here")
}
g.mu.Lock()
c, ok := g.computed, g.hasComputed
g.mu.Unlock()
if !ok {
return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet")
}
g.keepMu.Lock()
defer g.keepMu.Unlock()
p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why)
if _, err := p.keepRevisionOf(ctx, c); err != nil {
return meshwants.KeepNowAnswer{Fault: err.Error()}, nil
}
rev, _, _ := g.keptNow()
return meshwants.KeepNowAnswer{Revision: &rev}, nil
}
// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now.
func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.KeepNowInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return p.keepNow(ctx, in)
},
}
}
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
// node's capability reading and the first branch of each registration.
func (p *passer) graphInputs() assigngraph.Inputs {
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
// (design 47 G §708).
if r := p.seats.Reader(); r != nil {
in.Definitions = r.Definitions()
}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
first := p.w.trunkName()
in.FirstBranch = func(string) string { return first }
if p.ends != nil {
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
}
return in
}
// capabilityDigests is each node's capability reading, by node: the statement its engine made of what it reads, and the
// capabilities its tool runner states with the fingerprint of the runner that announced them, as the node's record carries
// them (design 46 approval mark §6, lead answer A-203), the digest over all of it.
func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem {
records := p.w.records()
known := p.w.knownNodes()
slices.Sort(known)
var out []meshwants.AssignmentRevisionCapabilitiesItem
for _, node := range known {
reads, read := p.reads.words(node)
rec, recorded := records[node]
if !read && !recorded {
continue
}
slices.Sort(reads)
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
Fingerprint: rec.RunnerFingerprint}
slices.Sort(item.Readings)
raw, _ := json.Marshal(struct {
Reads []string `json:"reads"`
Readings []string `json:"readings"`
Fingerprint string `json:"fingerprint"`
}{reads, item.Readings, item.Fingerprint})
sum := sha256.Sum256(raw)
item.Digest = "sha256:" + hex.EncodeToString(sum[:])
out = append(out, item)
}
return out
}
// sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is
// said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there.
func (p *passer) sayWanted(node, module, fp, pipeline string, removed bool) {
if p.publishWants == nil {
return
}
g := p.graph
var kept meshwants.AssignmentRevision
var source int64
ok := false
if g != nil {
kept, source, ok = g.keptNow()
}
if !ok {
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
return
}
at := p.clock().UTC()
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: kept.Revision, Source: source, At: &at}
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, kept.Revision, at.Format(time.RFC3339Nano))
if err := p.wantsSay(event, body, id); err != nil {
p.logf("wanted of %s on %s is not said: %v", module, node, err)
}
}
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
// version and the parts of the `version-released` whose parts are the digests its build.ended made (A-421): the catalog stamps no
// source on an entry (design 47 H §764, H-A-48). A trunk entry whose release is not heard is said, once, naming which was tried.
func (p *passer) namedBuild(module, trunkFP string, unreleased ...bool) func(fp string) (assign.KeptBuild, bool) {
return func(fp string) (assign.KeptBuild, bool) {
if k, ok := p.w.ends.partsOf(module, fp); ok {
return k, true
}
if len(unreleased) > 0 && unreleased[0] && fp == trunkFP {
// Wanted with no passed pipeline (the holder of mesh-catalog, the runner's first start): no pipeline released it,
// so its want names the parts its build.ended made, until a release of it is heard (design 47 G §16, A-428).
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
return k, true
}
if made := p.w.ends.madeOf(module, fp); len(made) > 0 {
return assign.KeptBuild{Parts: made}, true
}
}
if fp != trunkFP {
// A want held back at an older build (rule 2: no pipeline passed at the trunk's newer fingerprint) names that build's
// release when one names exactly the digests it made (novox 2026-11-16: mesh-hand-acts 0.1.13 held at 4eb3cf48 after
// the mesh-sdk merge moved its fingerprint).
return p.w.released.withParts(module, p.w.ends.madeOf(module, fp))
}
if _, _, kept := p.graphKept(); !kept {
p.unheardRelease(module, fp, "no assignment graph revision is kept yet, so no release is looked up")
return assign.KeptBuild{}, false
}
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
return k, true
}
p.unheardRelease(module, fp, "no version-released of it names the parts its build.ended made")
return assign.KeptBuild{}, false
}
}
// unheardRelease says once per module, fingerprint and why that a trunk entry's release is not heard, so its wants and candidates
// carry no version and no parts (design 47 G A-419).
func (p *passer) unheardRelease(module, fp, why string) {
key := "release " + module + "@" + fp + ": " + why
if p.skipped[key] {
return
}
if p.skipped == nil {
p.skipped = map[string]bool{}
}
p.skipped[key] = true
if p.logf != nil {
p.logf("the release of %s at fingerprint %s is not heard: %s; its wants and candidates carry no version and no parts", module, fp, why)
}
}
// graphKept is the revision the wants follow, with its trunk source, and whether one is kept; with no graph kept, none is.
func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) {
if p.graph == nil {
return meshwants.AssignmentRevision{}, 0, false
}
return p.graph.keptNow()
}
// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer
// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module).
func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) {
g := p.graph
g.mu.Lock()
g.failures++
n, retry, after := g.failures, g.retry, g.after
g.mu.Unlock()
if retry <= 0 {
retry = defaultKeepingRetry
}
if after == nil {
after = func(d time.Duration, f func()) { time.AfterFunc(d, f) }
}
p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err)
if n == keepingFailedAt && p.findings.Put != nil {
if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", kindKeepingFailed, err)
}
}
after(retry, func() {
g.mu.Lock()
newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision)
if !newer {
g.handedRev, g.handed = c.Revision, true
}
g.mu.Unlock()
if !newer {
p.needKeeping(ctx, keepNeed{c: &c})
}
})
}
// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared.
func (p *passer) keepingSucceeded() {
g := p.graph
g.mu.Lock()
n := g.failures
g.failures = 0
g.mu.Unlock()
if n >= keepingFailedAt && p.findings.Delete != nil {
o := keepingFailedFinding(n, nil)
if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil {
p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err)
}
}
}
// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)).
func keepingFailedFinding(n int, err error) assign.Observed {
summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n)
if err != nil {
summary += ": " + err.Error()
}
return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed,
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept",
Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.",
Source: "assigner." + kindKeepingFailed}
}
@@ -1,369 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// said is one event the passer said on the seat mesh-wants.
type said struct {
event, id string
body json.RawMessage
}
// Design 47 G A-67, §292: the pass keeps an assignment graph revision per catalog revision taken in, before it writes any want, and
// says assignment-revision; A-61, §693: a want written or removed is said `wanted` on the subject of its node and module in the same
// write, from the kept revision and its catalog's source, a put that wrote nothing says nothing, and with no revision kept none is said.
func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
kv := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): kv}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
placeOn(t, p.w, "ace", "web")
// Design 46 approval mark §6: a node's reading keeps the fingerprint of the runner that announced it, as its record says.
rec, _ := json.Marshal(nodeengine.HaveByNodeNodeValue{V: 1, Record: &nodeengine.NodeRecord{V: 1, Node: "ace",
RunnerCapabilities: []string{"bus-state-calls", "approval-mark/1"}, RunnerFingerprint: "sha256:runner"}})
p.w.put(buckets.Have, "ace.node", rec)
if c := p.capabilityDigests(); len(c) != 1 || c[0].Node != "ace" || c[0].Fingerprint != "sha256:runner" ||
!slices.Equal(c[0].Readings, []string{"approval-mark/1", "bus-state-calls"}) || !strings.HasPrefix(c[0].Digest, "sha256:") {
t.Errorf("the capability readings are %+v", c)
}
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fpTest}}}
objects := memObjects{}
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} }}
want := mwc.Want{Fp: fpTest, Self: true}
p.putWant(ctx, "ace", "web", want) // no revision kept yet: written, and not said
if len(events) != 0 || kv.puts != 1 {
t.Fatalf("with no revision kept: %d events, %d puts", len(events), kv.puts)
}
key, _ := mwc.WantKey("ace", "web")
delete(p.w.read(buckets.Wants()), key) // the world forgets the want, as before the first pass
kv.puts = 0
p.w.put(buckets.Wants(), key, nil)
p.keepGraph(ctx)
if len(events) != 1 || events[0].event != meshwants.EventAssignmentRevision || len(objects) == 0 {
t.Fatalf("the pass kept %d objects and said %+v", len(objects), events)
}
p.keepGraph(ctx) // the same inputs: nothing kept, nothing said
if len(events) != 1 {
t.Fatalf("a pass of the same inputs said %d events", len(events))
}
p.putWant(ctx, "ace", "web", want)
p.putWant(ctx, "ace", "web", want) // unchanged: not written, not said
if len(events) != 2 || kv.puts != 1 {
t.Fatalf("a want put twice: %d events, %d puts", len(events), kv.puts)
}
if wantEvent := "wanted.ace.web"; events[1].event != wantEvent {
t.Errorf("wanted said on %q, want %q", events[1].event, wantEvent)
}
var w meshwants.WantedEvent
if err := json.Unmarshal(events[1].body, &w); err != nil || w.Node != "ace" || w.Module != "web" || w.Fp != fpTest || w.Removed || w.Revision != 1 || w.Source != 9 {
t.Errorf("wanted = %+v (%v)", w, err)
}
p.putWant(ctx, "ace", "web", mwc.Want{Removed: true})
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
}
// Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a
// want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands.
t.Run("a want's run", func(t *testing.T) {
wkey, _ := mwc.WantKey("g14", "db")
stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w }
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
if w := stored(); w.Run != "12" || w.Pipeline != "main.12" {
t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w)
}
puts := kv.puts
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1})
if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts {
t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts)
}
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
if kv.puts != puts {
t.Error("a want that stands was written again")
}
skey, _ := mwc.WantKey("shanks", "db")
p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"})
if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" {
t.Errorf("a put's want is %+v, want the newest source held, 9", w)
}
// a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"})
if w := stored(); w.Run != "12" || w.Version != "0.1.0" {
t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w)
}
})
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
// keeps none, and the run's group-ended drops it.
before, puts := len(events), kv.puts
ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5}
var asked []meshcatalog.RevisionInput
call := func(key string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.RevisionInput)
asked = append(asked, in)
rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}}
a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}}
return json.Marshal(a)
}
if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 {
t.Errorf("a try's candidate asked %v: %v", asked, err)
}
if err := p.keepCandidate(ctx, call, ev); err != nil {
t.Fatal(err)
}
if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 {
t.Errorf("the candidate's revision was asked as %+v", asked)
}
a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"})
if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 {
t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err)
}
if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the run's entry of web is %+v: %v", e.Entry, err)
}
if len(events) != before || kv.puts != puts {
t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts)
}
if err := p.dropCandidate(ctx, "run-7"); err != nil {
t.Fatal(err)
}
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok {
t.Error("the run's group-ended left its candidate kept")
}
if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil {
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
}
}
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
type blockedObjects struct {
memObjects
mu sync.Mutex
gate chan struct{}
}
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
b.mu.Lock()
gate := b.gate
b.mu.Unlock()
if gate != nil {
<-gate
}
b.mu.Lock()
defer b.mu.Unlock()
return b.memObjects.Names(ctx)
}
func (b *blockedObjects) Put(ctx context.Context, name string, data []byte) error {
b.mu.Lock()
defer b.mu.Unlock()
return b.memObjects.Put(ctx, name, data)
}
func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.memObjects.Get(ctx, name)
}
// block stands a gate, opened by the function it answers.
func (b *blockedObjects) block() func() {
gate := make(chan struct{})
b.mu.Lock()
b.gate = gate
b.mu.Unlock()
return func() {
b.mu.Lock()
b.gate = nil
b.mu.Unlock()
close(gate)
}
}
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
// more. Each pass and each keeping says its duration.
func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
var mu sync.Mutex
var journal strings.Builder
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
said := func(words string) bool {
mu.Lock()
defer mu.Unlock()
return strings.Contains(journal.String(), words)
}
mdigest := strings.Repeat("e", 64)
mod, _ := mcc.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: mcc.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
placeOn(t, p.w, "ace", "web")
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 1, mpr.StepResultOutcomePassed)))
objects := &blockedObjects{memObjects: memObjects{}}
var kept sync.WaitGroup
woke := make(chan struct{}, 8)
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} },
background: func(f func()) { kept.Go(f) }, wake: func() { woke <- struct{}{} }}
pass := func() {
t.Helper()
done := make(chan struct{})
go func() { p.pass(context.Background()); close(done) }()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatal("the pass waits on the keeping of the assignment graph")
}
}
wantOf := func(node string) (mwc.Want, error) {
key, _ := mwc.WantKey(node, "web")
return mwc.DecodeWant(wantsKV.values[key])
}
open := objects.block()
t.Cleanup(func() { // the keeping ends before the test does, and says its duration
objects.mu.Lock()
standing := objects.gate != nil
objects.mu.Unlock()
if standing {
open()
}
kept.Wait()
if !said("the keeping of the assignment graph took ") {
t.Error("the keeping does not say its duration")
}
})
pass() // the start: the first keeping is blocked
if _, err := wantOf("ace"); err == nil {
t.Error("a pass at the start wrote a want before an assignment graph revision is kept")
}
open()
select {
case <-woke:
case <-time.After(10 * time.Second):
t.Fatal("the first keeping's end wakes no pass")
}
pass()
if w, err := wantOf("ace"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
t.Errorf("the want of web on ace = %+v (%v), want the passed version once a revision is kept", w, err)
}
open = objects.block() // a keeping of new inputs is blocked
placeOn(t, p.w, "g14", "web")
pass()
pass() // the same inputs again
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
}
p.graph.mu.Lock()
keeping, waiting := p.graph.keeping, p.graph.need
p.graph.mu.Unlock()
if !keeping || waiting != nil {
t.Errorf("keeping %v, need waiting %+v: the pass of the same inputs handed its revision again", keeping, waiting)
}
if !said("the pass took ") {
t.Error("the pass does not say its duration")
}
open()
kept.Wait()
// Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third
// failure in a row, cleared at the next that succeeds.
t.Run("a failed keeping runs again", func(t *testing.T) {
failing := &failingObjects{blockedObjects: objects, fail: 3}
var retries []time.Duration
var put []mcseat.Finding
var deleted []string
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil },
Delete: func(key string) error { deleted = append(deleted, key); return nil }}
p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second
p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() }
placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed
pass()
if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" ||
len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 {
t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures)
}
if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 {
t.Fatal("the revision is not kept once the store takes it")
}
// keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why.
if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil {
t.Error("a keeping by hand with no why runs")
}
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" {
t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err)
}
failing.fail, retries = 2, nil
p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) }
placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand
pass()
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 {
t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries)
}
var asked []string
tool := keepNowTools(func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
return json.RawMessage(`{}`), nil
})[0]
if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") ||
asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` {
t.Errorf("assigner_keep_now: %v, asked %v", err, asked)
}
})
}
// failingObjects refuses the next fail puts, then is objects.
type failingObjects struct {
*blockedObjects
fail int
}
func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error {
if f.fail > 0 {
f.fail--
return errors.New("the store answers nothing")
}
return f.blockedObjects.Put(ctx, name, data)
}
@@ -1,266 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"slices"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the
// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name
// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each
// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings;
// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
// half, never its own output back (§195).
type halves struct {
w *world
call mcc.Call
reader *views.ProcessReader
wake func()
logf func(string, ...any)
ctx context.Context
branches *mcc.Branches
mu sync.Mutex
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
settled chan<- struct{}
told bool
// regMu orders the registrations told to registered: the ones held when it is set, then each change (H-A-270 (3)).
regMu sync.Mutex
registered func(reg meshcatalog.Registration, gone bool)
}
// follow makes the branch follower on the reader; the reader's start asks the first views.
func (h *halves) follow() error {
b, err := mcc.FollowBranchesProcess(h.reader, h.call)
if err != nil {
return err
}
b.Logf = h.logf
b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
}
b.Adopted = h.adopted
// The one handler of registration-changed is the follower's (H-A-270 (3)): the pipeline ends are told each change from it,
// never from a second handler on the hub.
reconcile := b.Registrations.Follower.Changed
b.Registrations.Follower.Changed = func(module string, value json.RawMessage, gone bool, at time.Time) {
if reconcile != nil {
reconcile(module, value, gone, at)
}
h.tellRegistration(module, value, gone)
}
h.mu.Lock()
h.branches = b
h.mu.Unlock()
h.settleWhenNoBranch()
return nil
}
// settleWhenNoBranch tells the first pass once, when the registrations and the gate's source are answered and name no branch: the
// trunk is none-yet and the world is empty, as the catalog client's Ready says it (design 47 G, A-372). A mesh that names a branch is
// told by adopted, when that branch's first view is answered.
func (h *halves) settleWhenNoBranch() {
if h.settled == nil {
return
}
h.mu.Lock()
b := h.branches
h.mu.Unlock()
go func() {
select {
case <-b.FirstViews(h.ctx):
case <-h.ctx.Done():
return
}
if len(b.Followed()) > 0 || b.Ready() != nil {
return
}
h.mu.Lock()
told := h.told
h.told = true
h.mu.Unlock()
if !told {
h.settled <- struct{}{}
}
}()
}
// first is the first watched branch followed, the trunk's: "" while none is.
func (h *halves) first() string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return ""
}
if f := b.Followed(); len(f) > 0 {
return f[0]
}
return ""
}
// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its
// first view.
func (h *halves) adopted(branch string, c *mcc.Catalog) {
first := h.first()
if first == "" {
return
}
h.w.setTrunk(first)
h.wake()
if branch != first || h.settled == nil {
return
}
h.mu.Lock()
told := h.told
h.told = true
h.mu.Unlock()
if told {
return
}
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
}()
}
// changed takes one entry of branch's half into the world, or a module taken out of it.
func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) {
defer h.wake()
if deleted {
h.w.dropModule(branch, module)
return
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(raw, &e); err != nil {
h.logf("the entry of %s on branch %q does not read: %v", module, branch, err)
return
}
v, n, m := fromEntry(e)
h.w.putModule(branch, module, v, n, m)
h.w.putEdges(branch, module, e.Edges)
}
// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of
// its words.
func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) {
v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages,
Path: e.Path, Repository: e.Repository, Verdict: e.Verdict}
n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires,
Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
for p := range e.Packages {
n.Packages = append(n.Packages, p)
}
slices.Sort(n.Packages)
if len(e.Manifest) == 0 {
return v, n, nil
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return v, n, nil
}
sum := sha256.Sum256(raw)
v.Manifest = "sha256:" + hex.EncodeToString(sum[:])
man, err := catalogue.Decode(raw)
if err != nil {
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
}
n.Serves = graph.ServesOf(man)
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
return v, n, &parsed{module: e.Module, manifest: man}
}
// views is the catalog graph revision held now of each followed branch whose first view is answered.
func (h *halves) views() map[string]mcc.Half {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
out := map[string]mcc.Half{}
if b == nil {
return out
}
for _, branch := range b.Followed() {
if c, ok := b.Catalog(branch); ok {
if half, ok := c.View(); ok {
out[branch] = half
}
}
}
return out
}
// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the
// order `watch` added it). Once registrations are held, a module none names has none: it is unlisted (H-A-144). Only before the
// first registrations are held do the gate's source's branches stand in (the branches followed then), in the order the source
// gives them, by their names escaped as its keys are (design 47 G §532, H-A-143, H-A-144).
func (h *halves) watchedOf(module string) []string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return nil
}
if b.Registrations != nil && len(b.Registrations.All()) > 0 {
if reg, ok := b.Registrations.Registration(module); ok {
return slices.Clone(reg.Branches)
}
return nil
}
out := b.Followed()
slices.SortStableFunc(out, func(x, y string) int { return strings.Compare(keys.EscapeName(x), keys.EscapeName(y)) })
return out
}
// tellRegistration tells registered of one registration the follower holds now, or of one removed.
func (h *halves) tellRegistration(module string, value json.RawMessage, gone bool) {
h.regMu.Lock()
defer h.regMu.Unlock()
if h.registered == nil {
return
}
reg := meshcatalog.Registration{Module: module}
if !gone {
if err := json.Unmarshal(value, &reg); err != nil {
h.logf("the registration of %s does not read: %v", module, err)
return
}
}
h.registered(reg, gone)
}
// followRegistrations tells registered of every registration the branch follower holds and then of each change it takes
// (design 47 G §353): the registrations are followed once, by the follower's one handler of registration-changed (H-A-270
// (3)). It says false when no follower is made.
func (h *halves) followRegistrations(registered func(meshcatalog.Registration, bool)) bool {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil || b.Registrations == nil {
return false
}
h.regMu.Lock()
defer h.regMu.Unlock()
h.registered = registered
for _, reg := range b.Registrations.All() {
registered(reg, false)
}
return true
}
@@ -1,107 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// Design 47 G A-372: the assigner's wait for its catalog half asks the catalog client's Ready as well: a mesh whose registrations and
// gate's source are answered and name no branch is none-yet, and the first pass is told so once; before they answer it is not told.
func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
call := func(key string, _ any) (json.RawMessage, error) {
if key == mcc.RegistrationsVerb {
return json.Marshal(seat.RegistrationsAnswer{Registrations: []seat.Registration{}})
}
return json.Marshal(map[string][]string{"branches": {}})
}
for _, answered := range []bool{false, true} {
settled := make(chan struct{}, 2)
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx, settled: settled}
if err := h.follow(); err != nil {
t.Fatal(err)
}
if answered {
for _, f := range reader.Followers() {
if err := f.Start(ctx); err != nil {
t.Fatal(err)
}
}
select {
case <-settled:
case <-ctx.Done():
t.Fatal("a mesh that names no branch never settled the first pass")
}
}
select {
case <-settled:
t.Errorf("answered %v: the first pass was told twice or before the first views", answered)
default:
}
}
}
// Design 47 H-A-143, H-A-144, G §532: a module's watched branches are its registration's, in its order; once registrations are held a
// module none names has none; before the first registrations the gate's source's branches stand in, in the source's order.
// H-A-270 (3): the pipeline ends are told the registrations by the same follower, whether set before its first view or after.
func TestAModulesWatchedBranchesAreItsRegistrations(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
for _, tc := range []struct {
name string
regs []seat.Registration
module string
want []string
}{
{"its registration's order", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "link2pay", []string{"nox-mesh", "main"}},
{"no registration names it", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "checker", nil},
{"before the first registrations, the source's", []seat.Registration{}, "checker", []string{"main", "master"}},
} {
t.Run(tc.name, func(t *testing.T) {
call := func(key string, _ any) (json.RawMessage, error) {
switch key {
case mcc.RegistrationsVerb:
return json.Marshal(seat.RegistrationsAnswer{Registrations: tc.regs})
case mcc.SourceVerb:
return json.Marshal(map[string][]string{"branches": {"master", "main"}})
}
return nil, errors.New("not answered here")
}
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx}
if err := h.follow(); err != nil {
t.Fatal(err)
}
var live, held []string
if !h.followRegistrations(func(reg seat.Registration, gone bool) { live = append(live, reg.Module) }) {
t.Fatal("no follower of the registrations")
}
for _, f := range reader.Followers()[:2] { // the registrations and the gate's source
if err := f.Start(ctx); err != nil {
t.Fatal(err)
}
}
h.followRegistrations(func(reg seat.Registration, gone bool) { held = append(held, reg.Module) })
var regs []string
for _, r := range tc.regs {
regs = append(regs, r.Module)
}
if !slices.Equal(live, regs) || !slices.Equal(held, regs) {
t.Errorf("the ends were told %v at the first view and %v held, want %v", live, held, regs)
}
if got := h.watchedOf(tc.module); !slices.Equal(got, tc.want) {
t.Errorf("the watched branches of %s are %v, want %v", tc.module, got, tc.want)
}
})
}
}
@@ -1,581 +0,0 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
const inputsTool = "assigner_inputs"
const (
// inputsKeptFor is how long an input is kept at the longest.
inputsKeptFor = 24 * time.Hour
// inputsSwept is how many of the oldest keys each new ask judges.
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
)
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
type inputAsk struct {
Digest string `json:"digest"`
At string `json:"at"`
}
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
// carries its time in the key itself.
func readAsk(raw []byte) (digest string, at time.Time) {
var a inputAsk
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
at, _ = time.Parse(inputsTime, a.At)
return a.Digest, at
}
digest = string(raw)
t, _, _ := strings.Cut(digest, ".")
at, _ = time.Parse(inputsTime, t)
return digest, at
}
// isDigest is whether a key is an input's own key: 64 hex digits.
func isDigest(key string) bool {
if len(key) != 64 {
return false
}
_, err := hex.DecodeString(key)
return err == nil
}
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
// inputsCap is the state inputs' cap (design 47 G §354's kept-data row: 64 MiB, the manifest's max-mib): at it, an ask is refused.
const inputsCap = 64 << 20
// inputKeeper keeps the inputs over the state and its object store.
type inputKeeper struct {
vals stateobjects.Values
now func() time.Time
logf func(string, ...any)
// used is how many bytes the state and its object store hold; nil judges no cap.
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
// first, so a sweep never finds the input unnamed.
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
}
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
digest := digestOf(body)
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
if err != nil {
return "", err
}
askKey := inputsAskPrefix + keys.EscapeName(ask)
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
have, err := k.vals.KV.Get(digest)
if err != nil {
return "", err
}
if have != nil {
return digest, nil
}
if k.used != nil {
used, err := k.used()
if err == nil && used+int64(len(body)) > inputsCap {
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
buckets.InputsState, inputsCap>>20, used, len(body))
} else if err != nil {
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
}
if err != nil {
_ = k.vals.KV.Delete(askKey)
return "", err
}
}
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
}
askKey := inputsAskPrefix + keys.EscapeName(ask)
e, err := k.vals.KV.Get(askKey)
if err != nil || e == nil {
return err
}
digest, _ := readAsk(e.Value)
if err := k.vals.KV.Delete(askKey); err != nil {
return err
}
named, err := k.named()
if err != nil {
return err
}
if named[digest] {
return nil // another running ask still names it
}
return k.deleteInput(digest)
}
// named is the inputs some ask's key names now.
func (k *inputKeeper) named() (map[string]bool, error) {
all, err := k.vals.KV.Keys()
if err != nil {
return nil, err
}
named := map[string]bool{}
for _, key := range all {
if !strings.HasPrefix(key, inputsAskPrefix) {
continue
}
o, err := k.vals.KV.Get(key)
if err != nil {
return nil, err
}
if o != nil {
d, _ := readAsk(o.Value)
named[d] = true
}
}
return named, nil
}
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
// ends its object. A key already gone leaves nothing to remove here; the start and assigner_inputs_drop remove what a key left.
func (k *inputKeeper) deleteInput(key string) error {
e, err := k.vals.KV.Get(key)
if err != nil {
return err
}
if err := k.vals.KV.Delete(key); err != nil {
return err
}
if e == nil {
return nil
}
if ref, ok := stateobjects.ReadRef(e.Value); ok {
if err := k.vals.Objects.Delete(ref.Object); err != nil {
return fmt.Errorf("the object %s of %s: %w", ref.Object, key, err)
}
}
return nil
}
// dropKeyGone removes every object of the state whose key is gone and answers their count and bytes (A-465).
func (k *inputKeeper) dropKeyGone() (dropped []stateobjects.Kept, bytes int64, err error) {
kept, err := k.vals.Kept()
if err != nil {
return nil, 0, err
}
names, err := k.vals.DropKeyGone()
gone := map[string]bool{}
for _, n := range names {
gone[n] = true
}
for _, o := range kept {
if gone[o.Name] {
dropped = append(dropped, o)
bytes += o.Size
}
}
return dropped, bytes, err
}
// dropAtStart removes the objects whose key is gone at the program's start, said with their count and bytes (A-465). An object
// store not on the bus yet is said and left.
func (k *inputKeeper) dropAtStart() {
dropped, bytes, err := k.dropKeyGone()
if err != nil {
k.logf("inputs: the objects of deleted keys are not all removed at the start (%d objects, %d bytes removed): %v", len(dropped), bytes, err)
return
}
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
}
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
// input is gone (A-465b).
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
type askAt struct {
key string
at time.Time
}
var asks []askAt
var inputs []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
e, err := k.vals.KV.Get(key)
if err != nil || e == nil {
continue
}
d, at := readAsk(e.Value)
if !isDigest(d) {
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
errs = append(errs, k.vals.KV.Delete(key))
}
}
asks = append(asks, askAt{key, at})
continue
}
if !isDigest(key) {
t, _, _ := strings.Cut(key, ".")
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
errs = append(errs, k.deleteInput(key))
}
continue
}
inputs = append(inputs, key)
}
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
if len(asks) > inputsSwept {
asks = asks[:inputsSwept]
}
for _, a := range asks {
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
errs = append(errs, k.vals.KV.Delete(a.key))
}
}
named, err := k.named()
if err != nil {
return errors.Join(append(errs, err)...)
}
for _, key := range inputs {
if !named[key] {
errs = append(errs, k.deleteInput(key))
}
}
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none: the digest itself.
func (k *inputKeeper) keyOf(digest string) (string, error) {
if !isDigest(digest) {
return "", nil
}
e, err := k.vals.KV.Get(digest)
if err != nil || e == nil {
return "", err
}
return digest, nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
func (k *inputKeeper) body(digest string) ([]byte, error) {
if len(digest) != 64 {
return nil, errors.New("input takes the input's sha256, 64 hex digits")
}
key, err := k.keyOf(digest)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
}
if key == "" {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
var text string
found, err := k.vals.Get(key, &text)
if err != nil {
return nil, fmt.Errorf("the input %s cannot be read: %w", key, err)
}
if !found {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
}
b, err := base64.StdEncoding.DecodeString(text)
if err != nil || digestOf(b) != digest {
return nil, fmt.Errorf("the input %s does not hold the bytes its digest names", key)
}
return b, nil
}
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
// verb answers the seat's verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
// size. A caller that does not acknowledge within `within` ends the send, said in logf.
func (k *inputKeeper) verb(line buscalls.ChunkLine, within time.Duration) process.SeatHandler {
return func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.InputInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
if !callToken.MatchString(in.Call) {
return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
}
digest := strings.TrimPrefix(strings.TrimSpace(in.Digest), "sha256:")
body, err := k.body(digest)
if err != nil {
return nil, err
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: in.Call}
total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
go func() {
if err := send.Send(line, body, within, k.logf); err != nil {
k.logf("the send %s of the input %s: %v", in.Call, digest, err)
}
}()
return meshwants.InputAnswer{Digest: digest, Chunks: total}, nil
}
}
// inputObjects is the object store beside the state inputs (`inputs-objects`, a state's own, §712), opened when first used: until
// the grants part has made it every call is process.ErrNotYet.
type inputObjects struct {
js jetstream.JetStream
module string
mu sync.Mutex
bus stateobjects.Objects
}
func (o *inputObjects) open() (stateobjects.Objects, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.bus != nil {
return o.bus, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object store of the state inputs is not open: %w", err)
}
o.bus = stateobjects.BusObjects{Store: store}
return o.bus, nil
}
func (o *inputObjects) MostBytes() int { return 0 }
func (o *inputObjects) Put(name string, data []byte) error {
b, err := o.open()
if err != nil {
return err
}
return b.Put(name, data)
}
func (o *inputObjects) Get(name string) ([]byte, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.Get(name)
}
func (o *inputObjects) Delete(name string) error {
b, err := o.open()
if err != nil {
return err
}
return b.Delete(name)
}
func (o *inputObjects) List() ([]stateobjects.Object, error) {
b, err := o.open()
if err != nil {
return nil, err
}
return b.List()
}
// inputsOver is the inputs the program keeps, over its bucket and its object store beside it; nil when the bucket is not opened.
func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, logf func(string, ...any)) *inputKeeper {
if kv == nil {
return nil
}
objects := &inputObjects{js: js, module: module}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
used := func() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
n := int64(st.Bytes())
objs, err := objects.List()
if err != nil && !errors.Is(err, process.ErrNotYet) {
return 0, err
}
for _, o := range objs {
n += o.Size
}
return n, nil
}
return &inputKeeper{vals: v, now: time.Now, logf: logf, used: used}
}
// inputSaid is one kept input as assigner_inputs lists it.
type inputSaid struct {
Key string `json:"key"`
Time string `json:"time"`
Digest string `json:"digest"`
Size int `json:"size"`
Input string `json:"input,omitempty"`
}
// inputsRead is the tool assigner_inputs over the kept inputs: every kept input with its size, or the one named by digest whole.
func inputsRead(k *inputKeeper) stdio.Tool {
return stdio.Tool{Name: inputsTool, Description: "The inputs the assigner keeps for the composition steps it asked a build seat to run (design 47 G §354): each " +
"with its write time, digest and size, or one by `digest` whole (base64). Read-only.",
Input: schema(map[string]string{"digest": "one input's sha256"}, nil), Run: func(args map[string]any) (any, error) {
want := arg(args, "digest")
all, err := k.vals.KV.Keys()
if err != nil {
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
}
slices.Sort(all)
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
for _, key := range all {
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
lastAsk[d] = at.Format(inputsTime)
}
}
}
out := []inputSaid{}
for _, key := range all {
d := key
if !isDigest(key) || (want != "" && d != want) {
continue
}
t := lastAsk[key]
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
}
said := inputSaid{Key: key, Time: t, Digest: d, Size: base64.StdEncoding.DecodedLen(len(text))}
if b, err := base64.StdEncoding.DecodeString(text); err == nil {
said.Size = len(b)
}
if want != "" {
said.Input = text
}
out = append(out, said)
}
gone := []map[string]any{}
var goneBytes int64
answer := map[string]any{"inputs": out}
kept, err := k.vals.Kept()
if err != nil {
// never an empty list for objects that could not be read (A-465)
answer["key-gone-error"] = fmt.Sprintf("the objects of the state inputs cannot be listed, so key-gone objects are not known: %v", err)
return answer, nil
}
for _, o := range kept {
if o.KeyGone {
gone = append(gone, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
goneBytes += o.Size
}
}
answer["key-gone"], answer["key-gone-bytes"] = gone, goneBytes
return answer, nil
}}
}
// inputsDropTool is the tool that removes the objects whose key is gone.
const inputsDropTool = "assigner_inputs_drop"
// inputsDrop is the tool assigner_inputs_drop (A-465): a hand act, recorded first, that removes the objects of the state inputs
// whose key is gone now and answers what it removed.
func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) stdio.Tool {
return stdio.Tool{Name: inputsDropTool, Description: "Remove the objects of the assigner's state inputs whose key is gone now (design 47 G A-465), " +
"and answer which and how many bytes. An object a key names is never removed. Recorded as a hand act; assigner_inputs lists them as key-gone.",
Input: schema(map[string]string{"why": "why the objects of deleted keys are dropped now"}, []string{"why"}), Run: func(args map[string]any) (any, error) {
why := strings.TrimSpace(arg(args, "why"))
if why == "" {
return nil, errors.New("say why the objects of deleted keys are dropped")
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: inputsDropTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "inputs-drop"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was removed: %w", err)
}
dropped, bytes, err := k.dropKeyGone()
out := []map[string]any{}
for _, o := range dropped {
out = append(out, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
}
return map[string]any{"dropped": out, "count": len(out), "bytes": bytes, "hand-act": taken.Log}, err
}}
}
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if runEnded != nil {
if err := runEnded(e); err != nil {
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
_ = m.Nak()
return
}
}
if err := k.ended(e.Ask); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
@@ -1,503 +0,0 @@
package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
// delErr, when set, fails the next Delete once.
delErr error
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (k *kvMem) Get(key string) (*stateobjects.Entry, error) {
k.mu.Lock()
defer k.mu.Unlock()
v, ok := k.vals[key]
if !ok {
return nil, nil
}
return &stateobjects.Entry{Value: v, Revision: k.revs[key]}, nil
}
func (k *kvMem) Create(key string, v []byte) error {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, 1
return nil
}
func (k *kvMem) Update(key string, v []byte, rev uint64) error {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, rev+1
return nil
}
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
if err := k.delErr; err != nil {
k.delErr = nil
return err
}
delete(k.vals, key)
return nil
}
func (k *kvMem) Keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
sort.Strings(out)
return out, nil
}
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
listErr error // List fails with it
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(name string) error {
o.mu.Lock()
defer o.mu.Unlock()
delete(o.objs, name)
return nil
}
func (o *objectsMem) List() ([]stateobjects.Object, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.listErr != nil {
return nil, o.listErr
}
var out []stateobjects.Object
for n, d := range o.objs {
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
}
return out, nil
}
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
func testKeeper(objs *objectsMem, now func() time.Time) *inputKeeper {
return &inputKeeper{vals: stateobjects.Values{KV: newKVMem(), Objects: objs, Store: "inputs-objects"}, now: now, logf: func(string, ...any) {}}
}
// Design 47 G §354, §712, A-237: an input of any size is kept as an object beside the state, and the seat mesh-wants' verb input
// answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, time.Now)
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := k.keep("ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := k.body(digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := k.verb(line, time.Second)
raw, _ := json.Marshal(meshwants.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshwants.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+strings.Repeat("b", 64)+`","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// assigner_inputs lists it with its size and gives one whole by digest
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["inputs"].([]inputSaid)) != 1 || list.(map[string]any)["inputs"].([]inputSaid)[0].Size != len(body) {
t.Errorf("assigner_inputs = %+v %v", list, err)
}
}
// §354: an input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, judged at each new ask.
func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
now := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, func() time.Time { return now })
d1, err := k.keep("ask/one", []byte(`{"heads":"abc"}`))
if err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("a kept input is not read: %v", err)
}
if err := k.ended("ask/other"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("the step-ended of another ask deleted it: %v", err)
}
if err := k.ended("ask/one"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err == nil {
t.Error("the input of an ended ask is still answered")
}
if keys, _ := k.vals.KV.Keys(); len(keys) != 0 {
t.Errorf("keys left after the step-ended: %v", keys)
}
d2, _ := k.keep("ask/two", []byte(`{"heads":"def"}`))
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask/three", []byte(`{"heads":"ghi"}`)); err != nil {
t.Fatal(err)
}
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
for _, c := range []struct {
name string
delErr error
runErr error
naked bool
runs int
deleted bool
}{
{name: "the run's end before the input's delete", runs: 1, deleted: true},
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
} {
t.Run(c.name, func(t *testing.T) {
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
k.vals.KV.(*kvMem).delErr = c.delErr
runs := 0
runEnded := func(e buildstep.StepEnded) error {
runs++
if _, err := k.body(d); err != nil {
t.Errorf("the run's end is written after the input's delete (%d)", runs)
}
return c.runErr
}
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
m := &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, runEnded, m, t.Logf)
if m.naked != c.naked || m.acked == c.naked {
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
}
if m.naked && c.delErr != nil { // the redelivery
m = &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
}
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
}
})
}
}
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
// too; the objects of keys another part deleted are removed by dropKeyGone and by the tool assigner_inputs_drop, which lists
// them as key-gone first, and never removes an object a key names.
func TestAnEndedInputEndsItsObjectAndTheDropToolRemovesWhatAKeyLeft(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
big := func(b byte) []byte { return bytes.Repeat([]byte{b}, 2<<20) }
if _, err := k.keep("ask-one", big('a')); err != nil {
t.Fatal(err)
}
if _, err := k.keep("ask-two", big('b')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 2 {
t.Fatalf("objects = %d, want 2", len(objs.objs))
}
if err := k.ended("ask-one"); err != nil || len(objs.objs) != 1 {
t.Fatalf("the step-ended left %d objects (%v), want 1", len(objs.objs), err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-three", big('c')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("after the sweep of the day-old input %d objects, want only the new one", len(objs.objs))
}
// An object whose key another part deleted is listed as key-gone and removed by the tool; a live one stays.
keys, _ := k.vals.KV.Keys()
for _, key := range keys {
if !strings.HasPrefix(key, inputsAskPrefix) {
if err := k.vals.KV.Delete(key); err != nil {
t.Fatal(err)
}
}
}
live := []byte(`{"heads":"x"}`)
if _, err := k.keep("ask-four", append(live, make([]byte, 2<<20)...)); err != nil {
t.Fatal(err)
}
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["key-gone"].([]map[string]any)) != 1 {
t.Fatalf("assigner_inputs key-gone = %+v %v, want the one object of the deleted key", list, err)
}
ask := func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"answer":{"log":"hand-acts"}}`), nil
}
if _, err := inputsDrop(k, ask).Run(map[string]any{}); err == nil {
t.Error("dropped with no reason")
}
out, err := inputsDrop(k, ask).Run(map[string]any{"why": "a key left its object"})
if err != nil || out.(map[string]any)["count"] != 1 || len(objs.objs) != 1 {
t.Fatalf("assigner_inputs_drop = %+v %v, %d objects left: want one removed and the live one kept", out, err, len(objs.objs))
}
// The start drops what a key left, said with its count and bytes.
if _, err := k.keep("ask-five", live); err != nil {
t.Fatal(err)
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) {
_ = k.vals.KV.Delete(key)
}
}
var logged []string
k.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
before := len(objs.objs)
k.dropAtStart()
if len(objs.objs) != 0 || before == 0 || len(logged) != 1 || !strings.Contains(logged[0], fmt.Sprintf("%d objects of deleted keys removed at the start", before)) || !strings.Contains(logged[0], " bytes") {
t.Errorf("the start left %d of %d objects and said %q: want every key-gone object removed and its count and bytes said", len(objs.objs), before, logged)
}
}
func keysOf(t *testing.T, k *inputKeeper) []string {
t.Helper()
keys, err := k.vals.KV.Keys()
if err != nil {
t.Fatal(err)
}
return keys
}
// Design 47 G A-465: assigner_inputs says why key-gone objects are not known when the objects cannot be listed, and never
// answers them as none.
func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}, listErr: errors.New("the object store does not answer")}
k := testKeeper(objs, time.Now)
out, err := inputsRead(k).Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
if msg, _ := m["key-gone-error"].(string); !strings.Contains(msg, "does not answer") {
t.Errorf("answer %+v: want key-gone-error naming the failure", m)
}
if _, has := m["key-gone"]; has {
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
}
}
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
body := bytes.Repeat([]byte{'n'}, 2<<20)
var digests []string
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
d, err := k.keep(ask, body)
if err != nil {
t.Fatal(err)
}
digests = append(digests, d)
}
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
}
if _, err := k.vals.KV.Get(digests[0]); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
t.Errorf("a key %q beside the one input", key)
}
}
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
t.Errorf("the ask does not name the digest it uses: %v", e)
}
for _, ask := range []string{"ask-1", "ask-2"} {
if err := k.ended(ask); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
}
}
if err := k.ended("ask-3"); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
}
// a day after its last write, a renewed write counted
if _, err := k.keep("ask-4", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-5", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-6", []byte("other")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Errorf("an input written again 20 hours ago was swept: %v", err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-7", []byte("another")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
}
// an input of the old form goes a day after the time its key carries, with its object, never earlier
old := "20261117T090000Z." + digestOf([]byte("old"))
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
t.Fatal(err)
}
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
t.Fatal(err)
}
if err := k.sweep(); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
}
if e, _ := k.vals.KV.Get(recent); e == nil {
t.Error("an input of the old form under a day old was deleted early")
}
// a new content the cap refuses says the fill; the same content, already kept, needs no room
full := newTestKeeperWithFill(objs, now, inputsCap-10)
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
t.Fatalf("a content that fits is refused: %v", err)
}
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
t.Errorf("the refusal does not name the fill: %v", err)
}
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
t.Errorf("the content already kept is refused at the cap: %v", err)
}
}
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
k := testKeeper(objs, func() time.Time { return now })
k.used = func() (int64, error) { return used, nil }
return k
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,12 +1,7 @@
package main
import (
"encoding/json"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"context"
"errors"
@@ -16,31 +11,23 @@ import (
"os"
"path/filepath"
"reflect"
"runtime"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
if viewsEmitter(nil, process.Account{}, nil, &heldWriter{}, t.Logf).Findings == nil {
t.Error("the assigner's views emitter writes no view-unsaid finding")
}
// 1297/B2a: without a credential the program does not start, and says why.
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
@@ -55,14 +42,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
var reads []string
for _, r := range m.Reads {
if slices.Contains(readFamilies, r) {
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
if slices.Contains(readLogs, r) {
continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket
}
reads = append(reads, strings.Replace(r, ".", "_", 1))
}
for _, l := range readFamilies {
for _, l := range readLogs {
if !slices.Contains(m.Reads, l) {
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
t.Errorf("the program reads the log %s, the manifest does not name it", l)
}
}
var states []string
@@ -72,7 +59,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, pair := range []struct {
what string
code, want []string
}{{"reads", readBuckets, reads}, {"states", append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()), states}} { // the marks are bound by the views family (views.go)
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
if !reflect.DeepEqual(got, want) {
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
@@ -82,28 +69,35 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, s := range keptStates {
kept = append(kept, m.Module+"_"+s.Name)
}
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()))); !reflect.DeepEqual(got, want) {
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
t.Errorf("the program announces %v, keeps %v", got, want)
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
for _, c := range []struct {
missing string
err error
starts bool
lease []byte
said string
}{
{buckets.Puts(), jetstream.ErrBucketNotFound, true, "no put is taken further"},
{buckets.Puts(), errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{buckets.Puts(), context.DeadlineExceeded, true, "no put is taken further"},
{buckets.Wants(), errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets())-1 || !strings.Contains(logged, c.said)):
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
@@ -114,21 +108,19 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
kvs := buckets{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
placed, err := mmac.AssignmentsValues(mma.Assignments{Node: "ace"})
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
for key, raw := range placed {
kv := kvs[plc.AssignmentsBucket()].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[buckets.Puts()].(*watchKV).closed = true
kvs[buckets.Wants()].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner"), log.New(io.Discard, "", 0), func(string) string { return "" })
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -144,13 +136,18 @@ type refusingJS struct {
jetstream.JetStream
missing string
err error
lease []byte // the controller's lease, when it holds one
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
return &memKV{values: map[string][]byte{}}, nil
kv := &memKV{values: map[string][]byte{}}
if name == buscompose.LeaseBucket && r.lease != nil {
kv.values[buscompose.ControllerLeaseKey] = r.lease
}
return kv, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
@@ -198,63 +195,3 @@ func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (e kvEntry) Revision() uint64 { return 1 }
func (e kvEntry) Created() time.Time { return time.Time{} }
// Keys lists nothing: the retired state this watch stands for holds no key.
func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
return nil, jetstream.ErrNoKeysFound
}
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
// raise a finding into the same map.
var putIntoMu sync.Mutex
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
putIntoMu.Lock()
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
putIntoMu.Unlock()
return err
}
}
// fakeWire is a views part's finding writer that hands each written finding's kind on.
type fakeWire struct{ written chan string }
func (f fakeWire) WriteWire(_ context.Context, finding map[string]any) error {
k, _ := finding["kind"].(string)
f.written <- k
return nil
}
func (fakeWire) Clear(context.Context, string, string, string, string) error { return nil }
func (fakeWire) ModuleNode() (string, string) { return "mesh-delivery-assigner", "n1" }
// H-A-60: the assigner's entries are asked again to the setting entry-asks, then written through its finding writer as the
// finding entry-unanswered.
func TestAnUnreachedEntryEndsInTheAssignersFindingEntryUnanswered(t *testing.T) {
ctx := context.Background()
unreached := func(string, any) (json.RawMessage, error) { return nil, errors.New("no responders") }
w := fakeWire{written: make(chan string, 4)}
e := entriesOf(nil, unreached, 1, w, func() {})
if e.Asks != 1 {
t.Fatalf("entry-asks %d, want 1 from the setting", e.Asks)
}
e.Need(ctx, "src2:u")
deadline := time.Now().Add(5 * time.Second)
for {
select {
case kind := <-w.written:
if kind != meshcatalogclient.EntryUnansweredKind {
t.Fatalf("the finding written is %q, want %s", kind, meshcatalogclient.EntryUnansweredKind)
}
return
default:
}
if time.Now().After(deadline) {
t.Fatal("no finding entry-unanswered is written past the bound")
}
e.AskAgain(ctx)
runtime.Gosched()
}
}
@@ -1,150 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
plc "mesh-delivery-assigner/internal/assignment"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
)
// The state this seat's assignments were kept in before ADR 0641 gave it its name. The assigner reads it here and nowhere
// else: at its start it copies each key into `assignments` (copyRetired) and keeps no reader of it afterwards. The
// manifest still declares the state, so the assigner's account may read it for the copy; a later release drops it.
const retiredState = "placement"
func retiredBucket() string { return plc.Module() + "_" + retiredState }
// CopiedEvent is the module's event saying the copy was made, once: mesh.mod.mesh-delivery-assigner.event.assignments-copied.
const CopiedEvent = "assignments-copied"
// copiedSaid is what the event carries: the keys copied, those already in `assignments` (kept as they are), and the keys of
// the form `placement.<node>` that were dropped (their node key holds the value).
type copiedSaid struct {
Copied int `json:"copied"`
Present int `json:"present"`
Dropped int `json:"dropped"`
Skipped int `json:"skipped"`
}
// copiesState is the assigner's own state of its copies, read by its tool assigner_read_copies, and copiesBucket its bucket.
// The retired state gets nothing written: readers of other modules still parse every key in it.
const copiesState = "copies"
func copiesBucket() string { return plc.Module() + "_" + copiesState }
// copiedMarker is the key, in the state copies, that says the copy ended: its time and the keys it copied. It is written
// after the last key, and the copy is made at every start until it is there and never after, so a copy cut short by a
// crash is completed at the next start and a key `assignments` dropped after the copy ended is never brought back.
const copiedMarker = "copied-from-retired"
type copiedMark struct {
At time.Time `json:"at"`
Copied int `json:"copied"`
}
// copyRetired is the holder's switch from the retired state to `assignments`, with no gap: until the marker is there, it
// creates in `assignments` each key of the retired state in the form `<node>.node` or `<node>.<module>` that is missing
// there (never overwriting one), the module keys first and the node keys, which commit a node, last, and then writes the
// marker. It deletes nothing: the readers delivered before this one still read the retired state, and a delete of a
// `<node>.node` key there is a node leaving (ADR 0641 section 4); the state is dropped in the next release. Keys of the
// form `placement.<node>` are not copied, and a key that fits no form is left and counted. It answers whether the seat may
// go on: a copy that failed is tried again at the next pass.
func (p *passer) copyRetired(ctx context.Context) bool {
if p.copied {
return true
}
old, next, copies := p.kvs[retiredBucket()], p.kvs[plc.AssignmentsBucket()], p.kvs[copiesBucket()]
if next == nil || copies == nil {
return false
}
if old == nil {
p.logf("assignments: the retired state %s cannot be read, so nothing is copied from it", retiredBucket())
p.copied = true
return true
}
switch _, err := copies.Get(ctx, copiedMarker); {
case err == nil:
p.copied = true // the copy ended at an earlier start: nothing is copied again
return true
case !errors.Is(err, jetstream.ErrKeyNotFound):
p.logf("assignments: the marker %s of %s cannot be read, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
names, err := old.Keys(ctx)
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
p.logf("assignments: the keys of %s cannot be listed, and are tried again at the next pass: %v", retiredBucket(), err)
return false
}
// A node's module keys before its node key: the node key is the commit the readers wait for.
slices.SortStableFunc(names, func(a, b string) int {
return strings.Compare(sortForm(a), sortForm(b))
})
said := copiedSaid{}
for _, key := range names {
_, _, kind, perr := mmac.ParseKey(key)
switch {
case strings.HasPrefix(key, retiredState+"."):
// The node named placement could not exist before, so this is the form that held a whole node in one key.
said.Dropped++
case perr != nil || (kind != mmac.NodeKind && kind != mmac.ModuleKind):
said.Skipped++
continue
default:
entry, err := old.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
p.logf("assignments: %s cannot be read, and the copy is tried again at the next pass: %v", key, err)
return false
}
if entry.Operation() != jetstream.KeyValuePut {
continue
}
rev, err := next.Create(ctx, key, entry.Value())
switch {
case errors.Is(err, jetstream.ErrKeyExists):
said.Present++
case err != nil:
p.logf("assignments: %s is not copied, and the copy is tried again at the next pass: %v", key, err)
return false
default:
said.Copied++
p.w.putEntry(plc.AssignmentsBucket(), key, entry.Value(), rev, p.clock())
}
}
}
mark, _ := json.Marshal(copiedMark{At: p.clock(), Copied: said.Copied + said.Present})
if _, err := copies.Put(ctx, copiedMarker, mark); err != nil {
p.logf("assignments: the end of the copy is not written as %s in %s, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
return false
}
p.copied = true
if said == (copiedSaid{}) {
return true // the retired state holds nothing
}
p.logf("assignments: copied %d keys of %s into %s (%d already there kept as they are, %d of the form placement.<node> not copied, %d fitting no form left)",
said.Copied, retiredBucket(), plc.AssignmentsBucket(), said.Present, said.Dropped, said.Skipped)
if p.seat != nil {
if err := p.seat.bus.Emit(CopiedEvent, said); err != nil {
p.logf("assignments: the copy is not said as an event: %v", err)
}
}
return true
}
// sortForm orders `<node>.<module>` keys before `<node>.node`.
func sortForm(key string) string {
if node, ok := strings.CutSuffix(key, ".node"); ok {
return node + "\x7f"
}
return key
}
@@ -1,26 +0,0 @@
package main
import (
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// The module claiming the seat node-engine is found by its served verbs, in the catalog's form `seat:<seat>.<verb>`
// and in the bare form (live, 2026-11-16: every composition step said "holds no module claiming the seat node-engine").
func TestTheNodeEngineEntryIsFoundByItsServedVerbs(t *testing.T) {
for _, c := range []struct {
name string
serves []string
found bool
}{
{"the catalog's seat form", []string{"seat:node-engine.compose", "seat:node-engine.acts"}, true},
{"the bare form", []string{"node-engine.compose"}, true},
{"another seat", []string{"seat:node-engine-x.compose", "seat:mesh-wants.ask"}, false},
} {
e, ok := nodeEngineEntry(map[string]meshcatalog.CatalogEntry{"other": {Serves: []string{"seat:mesh-builds.ask"}}, "engine": {Serves: c.serves}})
if ok != c.found || (ok && e.Serves[0] != c.serves[0]) {
t.Errorf("%s: found %v (%v), want %v", c.name, ok, e.Serves, c.found)
}
}
}
@@ -1,155 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assigngraph"
)
// busObjects is the assigner's own object bucket assignment-revisions in the old `objects` section (novox/hq ADR 0619 §2),
// which the grants part creates, as the passer's process reaches it: opened at its first use and kept; until the bucket exists
// every call is an error saying so, and the next call opens it again (design 47 G A-67, modelled on the indexer's busObjects).
type busObjects struct {
js jetstream.JetStream
module string
local string
mu sync.Mutex
o jetstream.ObjectStore
}
// newBusObjects is the object bucket of this local name of the module the account runs as.
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
return &busObjects{js: js, module: cred.Module, local: local}
}
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.o != nil {
return b.o, nil
}
o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(b.module, b.local))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
}
if err != nil {
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
}
b.o = o
return o, nil
}
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
o, err := b.open(ctx)
if err != nil {
return err
}
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
return err
}
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
data, err := o.GetBytes(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return nil, assigngraph.ErrNoObject
}
return data, err
}
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
infos, err := o.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return []string{}, nil
}
if err != nil {
return nil, err
}
names := make([]string, 0, len(infos))
for _, i := range infos {
names = append(names, i.Name)
}
return names, nil
}
// PutDescribed stores data under name with the entry's module and fingerprint in its metadata, which the listing says (assigngraph.Describer,
// design 47 H-A-259).
func (b *busObjects) PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name, Metadata: map[string]string{entryModuleMeta: module, entryFpMeta: fp}}, bytes.NewReader(data))
return err
}
// Described is every object with the module and fingerprint its metadata says, in one listing.
func (b *busObjects) Described(ctx context.Context) ([]assigngraph.Described, error) {
o, err := b.open(ctx)
if err != nil {
return nil, err
}
infos, err := o.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return []assigngraph.Described{}, nil
}
if err != nil {
return nil, err
}
out := make([]assigngraph.Described, 0, len(infos))
for _, i := range infos {
out = append(out, assigngraph.Described{Name: i.Name, Module: i.Metadata[entryModuleMeta], Fp: i.Metadata[entryFpMeta]})
}
return out, nil
}
// The metadata keys of an entry object's module and fingerprint.
const (
entryModuleMeta = "module"
entryFpMeta = "fp"
)
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
func (b *busObjects) Delete(ctx context.Context, name string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
o, err := b.open(ctx)
if err != nil {
return 0, 0, err
}
st, err := o.Status(ctx)
if err != nil {
return 0, 0, err
}
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
if err != nil {
return int64(st.Size()), 0, err
}
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
}
@@ -1,460 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The wait on an outside cause (design 47 G §706, ADR 0677): the assigner judges a node's health after a want moves, and a moved
// module's failing check on a machine whose open conditions name an outside cause is held as a wait (internal/outside has the
// rules). The conditions come only through the seat mesh-conditions: its verb `conditions`, read at each judging as a kept
// call, and its events, which only say when to judge again.
// conditionsVerb is the seat verb the keeper answers `conditions` by.
const conditionsVerb = "seat:" + mcseat.Seat + "." + mcseat.VerbConditions
// wantsPublisher says one of the seat mesh-wants' events, `wait-held` and `wait-ended` among them, under a message id the bus
// keeps one of.
func wantsPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(seatdef.EventSubject(meshwants.Seat, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", assignment.Module())
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
return err
}
}
// outsideActs keeps, while the process runs, what the judging of outside waits needs beside the kept waits: the conditions read
// for each machine (one answer is one judging), the machines a heard event says to judge again, the machines whose own network
// is held open, and each check's last signature.
type outsideActs struct {
mu sync.Mutex
call func(key string, body any) (json.RawMessage, error) // a kept call
wake func()
logf func(string, ...any)
// bound and settle are the settings outside-wait-minutes and outside-settle-seconds.
bound, settle time.Duration
// reading is a read in flight by machine, answered the answers it brought, not yet judged by.
reading map[string]bool
answered map[string][]outside.Cond
// dirty is a machine whose waits are judged again, cleared a wait's key whose cause's clearing was heard.
dirty map[string]bool
cleared map[string]bool
// sigs is each check's last signature, by node.module.
sigs map[string]string
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
causes map[string][]string
// started says the first pass judged every wait again.
started bool
}
func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func(), logf func(string, ...any), bound, settle time.Duration) *outsideActs {
if bound <= 0 {
bound = outside.DefaultBound
}
if settle <= 0 {
settle = outside.DefaultSettle
}
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, causes: map[string][]string{}}
}
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
// each read at the start from the file the manifest declares with the content ${config:<key>}, which the environment names
// (A-112, as the builder reads its own); zero for one whose file is missing or empty, which is its default, and an error
// naming the file for a content that is no positive whole number.
func outsideSettings(getenv func(string) string) (bound, settle time.Duration, err error) {
minutes, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_WAIT_FILE"))
if err != nil {
return 0, 0, err
}
seconds, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"))
if err != nil {
return 0, 0, err
}
return time.Duration(minutes) * time.Minute, time.Duration(seconds) * time.Second, nil
}
// settingFile reads one setting's file: 0 for no path, no file or an empty one.
func settingFile(path string) (int, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
}
return n, nil
}
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again; nothing is kept
// of the condition (the network hold ends, design 47 H §750, H-A-55).
func (o *outsideActs) Heard(subject string, body []byte) {
var e mcseat.ConditionEvent
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
o.logf("%s holds what is not a condition's event, and is dropped: %v", subject, err)
return
}
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
o.mu.Lock()
if cleared {
for _, node := range o.causes[e.Key] {
o.dirty[node] = true
o.cleared[node+"|"+e.Key] = true
}
} else if e.Subject != nil {
o.dirty[e.Subject.Machine] = true
for _, m := range e.Subject.Also {
o.dirty[m] = true
}
}
o.mu.Unlock()
o.wake()
}
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
// which it starts when none is in flight.
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
o.mu.Lock()
defer o.mu.Unlock()
if c, ok := o.answered[machine]; ok {
delete(o.answered, machine)
return c, true
}
if o.reading[machine] {
return nil, false
}
o.reading[machine] = true
go func() {
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine", "machine": machine})
var conds []outside.Cond
if err == nil {
conds, err = outside.ParseConditions(answer)
}
o.mu.Lock()
o.reading[machine] = false
if err != nil {
o.logf("the conditions of %s are not read (%v): the move waits on %s", machine, err, outside.OnUnread)
} else {
o.answered[machine] = conds
o.dirty[machine] = true // the kept call answered: the move is judged by it
}
o.mu.Unlock()
if err == nil {
o.wake()
}
}()
return nil, false
}
// outsideReview is the judging of module's check at fp on each of its nodes (§706): it returns the nodes as the rules read them,
// the have of a waiting move no failure (its health unknown) and a go-on's counted healthy, and keeps the waits, says their
// events, finding and ask. moved says the delivery moved owner to node.
func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []string, n assign.Nodes, keeperSelf bool, moved func(owner, node string) bool) assign.Nodes {
o := p.outside
if o == nil || fp == "" || p.kvs[buckets.Waits()] == nil {
return n
}
hex, err := catalogkey.KeyPart(fp)
if err != nil {
return n
}
now := p.clock()
kept := p.w.read(buckets.Waits())
override := map[string]string{}
// A wait of another fingerprint of this module is replaced by the newer want.
for key, raw := range kept {
if w, err := outside.Decode(raw); err == nil && w.Module == module && w.FP != fp {
p.outsideEnd(ctx, key, w, outside.HowReplaced, now)
}
}
for _, node := range nodes {
key := outside.Key(node, module, hex)
var prev *outside.Wait
if raw := kept[key]; raw != nil {
if w, err := outside.Decode(raw); err == nil {
prev = &w
}
}
h, ok := n.Haves[node][module]
sig := fmt.Sprintf("%s|%s|%s", h.Fp, h.Phase, h.Health)
o.mu.Lock()
first := !o.started
changed := o.sigs[node+"."+module] != sig
o.sigs[node+"."+module] = sig
dirty := o.dirty[node]
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
o.mu.Unlock()
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
if !failing && !healthy && prev == nil {
continue
}
if prev != nil && p.outsideAnswered(prev, node, module, fp) {
dirty = true
}
due := prev != nil && (prev.Release != "" || (prev.Settling && !now.Before(prev.SettleEnds)))
var d outside.Decision
if prev != nil && !(first || changed || dirty || due) {
// nothing to judge by: the wait stands as kept; the bound's ask is made when it has run
d = outside.Decision{Wait: prev, Waiting: prev.Standing() || prev.Settling, Own: prev.Decided == outside.HowFailed || prev.Decided == outside.HowOwnFailure,
Counted: prev.Decided == outside.HowGoOn}
if prev.Standing() && !now.Before(prev.ClockFrom.Add(o.bound)) {
d.Ask = true
}
} else {
f := outside.Facts{Now: now, Failing: failing, Healthy: healthy, Node: node, KeeperSelf: keeperSelf, Bound: o.bound, Settle: o.settle,
Moved: func(owner string) bool { return moved(owner, node) }, CauseCleared: cleared}
if failing && !keeperSelf {
f.Conds, f.Read = o.take(node)
}
d = outside.Judge(prev, node, module, fp, f)
o.mu.Lock()
delete(o.dirty, node)
if prev != nil {
delete(o.cleared, node+"|"+prev.Cause)
}
o.mu.Unlock()
}
switch {
case d.Waiting:
override[node] = nodeengineclient.HealthUnknown
case d.Counted:
override[node] = nodeengineclient.HealthOK
}
p.outsideApply(ctx, key, hex, node, module, fp, prev, d, now)
}
o.mu.Lock()
o.started = true
o.mu.Unlock()
if len(override) == 0 {
return n
}
out := n
out.Haves = map[string]map[string]nodeengine.HaveByNodeModuleValue{}
for node, byModule := range n.Haves {
out.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
for m, h := range byModule {
out.Haves[node][m] = h
}
}
for node, health := range override {
h := out.Haves[node][module]
h.Health = health
if health == nodeengineclient.HealthOK {
h.Phase = nodeengineclient.PhaseRunning
}
out.Haves[node][module] = h
}
return out
}
// outsideAnswered reads the operator's answer to the wait's ask, from the teller's record of it under the ask's key (answeredWith,
// §700's path as the assigner reads an ask's answer), and notes it on the wait as a release; it says whether it did.
func (p *passer) outsideAnswered(w *outside.Wait, node, module, fp string) bool {
if w.Ask == "" || w.Release != "" {
return false
}
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return false
}
for _, how := range []string{assign.OptionGoOn, assign.OptionFailed} {
if p.answeredWith(key, how) {
w.Release, w.By, w.Why = how, "the operator", "the answer to the ask "+w.Ask
return true
}
}
return false
}
// outsideApply keeps what a judging came to: the wait's record, its events, its finding and its ask.
func (p *passer) outsideApply(ctx context.Context, key, hex, node, module, fp string, prev *outside.Wait, d outside.Decision, now time.Time) {
o := p.outside
switch {
case d.End != "":
p.outsideEnd(ctx, key, *prev, d.End, now)
if d.Wait != nil {
p.keepWait(ctx, key, *d.Wait)
}
return
case d.Wait == nil:
if prev != nil {
p.outsideEnd(ctx, key, *prev, "", now)
}
return
}
w := *d.Wait
if d.Held {
body := p.outsideBody(w)
if err := p.wantsSay(meshwants.EventWaitHeld, body, key+"."+w.On+"."+w.ClockFrom.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the wait of %s on %s is not said, and is said at the next pass: %v", module, node, err)
return
}
p.outsideFinding(w, true)
}
if d.Ask && w.Ask == "" {
if id := p.outsideAsk(ctx, node, module, fp, w, now); id != "" {
w.Ask = id
}
}
if w.Standing() || w.Settling || w.Decided != "" {
o.mu.Lock()
if w.Cause != "" && w.Standing() {
if !containsString(o.causes[w.Cause], node) {
o.causes[w.Cause] = append(o.causes[w.Cause], node)
}
}
o.mu.Unlock()
if w.Settling {
p.wakeAt(w.SettleEnds, now)
} else if w.Standing() {
p.wakeAt(w.ClockFrom.Add(o.bound), now)
}
p.keepWait(ctx, key, w)
}
}
func containsString(l []string, s string) bool {
for _, x := range l {
if x == s {
return true
}
}
return false
}
// keepWait writes the wait's record.
func (p *passer) keepWait(ctx context.Context, key string, w outside.Wait) {
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return outside.Encode(w) })
}
func (p *passer) outsideBody(w outside.Wait) meshwants.Wait {
since, from := w.Since, w.ClockFrom
return meshwants.Wait{Step: meshwants.WaitStepOutside, Owner: assignment.Module(), Node: w.Node, Module: w.Module, Fingerprint: w.FP, On: w.On,
Since: since, ClockFrom: &from, Bound: w.Bound,
Words: fmt.Sprintf("%s on %s waits on %s", w.Module, w.Node, w.On)}
}
// outsideEnd ends a wait: its wait-ended with how (none for a settle, which was never said), its finding and ask taken away, and
// its record deleted unless the end decided it (the caller keeps that record).
func (p *passer) outsideEnd(ctx context.Context, key string, w outside.Wait, how string, now time.Time) {
if w.Standing() && how != "" {
body := p.outsideBody(w)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+"."+how+"."+now.UTC().Format(time.RFC3339Nano)); err != nil {
p.logf("the end of the wait of %s on %s is not said, and is said at the next pass: %v", w.Module, w.Node, err)
return
}
}
p.outsideFinding(w, false)
if ask, err := asks.OutsideAskKey(w.Node, w.Module, w.FP); err == nil {
p.remove(ctx, buckets.Asks(), ask)
}
if p.outside != nil && w.Cause != "" {
p.outside.mu.Lock()
p.outside.causes[w.Cause] = removeString(p.outside.causes[w.Cause], w.Node)
p.outside.mu.Unlock()
}
p.remove(ctx, buckets.Waits(), key)
}
func removeString(l []string, s string) []string {
var out []string
for _, x := range l {
if x != s {
out = append(out, x)
}
}
return out
}
// outsideFinding writes the finding delivery-waits-outside, or clears it.
func (p *passer) outsideFinding(w outside.Wait, raised bool) {
if p.findings.Put == nil {
return
}
o := assign.WaitsOutsideFinding(w.Node, w.Module, w.Cause, w.On, p.clock())
if raised {
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", "delivery."+o.ID, err)
}
return
}
if p.findings.Delete != nil {
if err := p.findings.Delete(mcseatKey(o)); err != nil {
p.logf("the finding %s is not cleared: %v", "delivery."+o.ID, err)
}
}
}
func mcseatKey(o assign.Observed) string {
return views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
}
// outsideAsk asks the operator at the wait's bound, once, and returns the asking's id.
func (p *passer) outsideAsk(ctx context.Context, node, module, fp string, w outside.Wait, now time.Time) string {
key, err := asks.OutsideAskKey(node, module, fp)
if err != nil {
return ""
}
asking := assign.AskingSeconds(now)
a := assign.OutsideAsk(key, asks.OutsideAbout(key), node, module, fp, w.On, w.ClockFrom, asking, now)
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
if ask, err := assign.AskOf(a); err == nil {
return ask.ID
}
return ""
}
// wakeAt makes a pass when t comes: the settle's end and the bound are the waits' own times, the one wake (no polling).
func (p *passer) wakeAt(t, now time.Time) {
if p.outside == nil || p.outside.wake == nil {
return
}
time.AfterFunc(max(t.Sub(now), 0)+time.Millisecond, p.outside.wake)
}
@@ -1,282 +0,0 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The whole of the wait on an outside cause in the pass (design 47 G §706): a fixture keeper answering `conditions` and saying
// its events, a failing check of nats on ace, and the events, finding, ask and record the pass leaves.
type outsideRig struct {
t *testing.T
p *passer
now time.Time
conds []map[string]any
unread bool
events []string
bodies []meshwants.Wait
found map[string]bool
woke chan struct{}
health string
}
func newOutsideRig(t *testing.T) *outsideRig {
r := &outsideRig{t: t, now: time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC), found: map[string]bool{}, woke: make(chan struct{}, 16), health: nodeengineclient.HealthFailed}
r.p = newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, r.now)
r.p.now = func() time.Time { return r.now }
r.p.w.put(buckets.Asks(), "x", nil)
r.p.outside = newOutsideActs(func(key string, body any) (json.RawMessage, error) {
if r.unread {
return nil, context.DeadlineExceeded
}
return json.Marshal(map[string]any{"conditions": r.conds})
}, func() { r.woke <- struct{}{} }, func(string, ...any) {}, 30*time.Minute, 90*time.Second)
r.p.publishWants = func(event string, body any, id string) error {
r.events = append(r.events, event)
r.bodies = append(r.bodies, body.(meshwants.Wait))
return nil
}
r.p.findings.Put = func(f mcseat.Finding) error { r.found[f.Scope+"."+f.ID+"."+f.Kind] = true; return nil }
r.p.findings.Delete = func(key string) error { delete(r.found, key); return nil }
return r
}
func (r *outsideRig) cond(key string, also ...string) {
r.conds = append(r.conds, map[string]any{"key": key, "subject": map[string]any{"machine": strings.Split(key, ".")[1], "also": also}})
}
// review is one pass's judging of nats on ace at the fingerprint, with the read answered at once when one is started.
func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) assign.Nodes {
r.t.Helper()
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
healthy := r.health == nodeengineclient.HealthOK
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
if moved == nil {
moved = func(string, string) bool { return false }
}
out := r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
select {
case <-r.woke: // the read was answered: the next pass judges by it
out = r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
case <-time.After(50 * time.Millisecond):
}
return out
}
func (r *outsideRig) wait() outside.Wait {
r.t.Helper()
for _, v := range r.p.w.read(buckets.Waits()) {
w, err := outside.Decode(v)
if err != nil {
r.t.Fatal(err)
}
return w
}
return outside.Wait{}
}
func TestTheWaitOnAnOutsideCause(t *testing.T) {
healthOf := func(n assign.Nodes) string { return n.Haves["ace"]["nats"].Health }
t.Run("a cause listing the machine is a wait said at once, ended cleared when it clears healthy", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 1 || r.events[0] != meshwants.EventWaitHeld {
t.Fatalf("health %s events %v", healthOf(n), r.events)
}
b := r.bodies[0]
if b.Step != meshwants.WaitStepOutside || b.On != "unreachable:hub" || b.Bound != "30m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(r.now) || b.Node != "ace" || b.Module != "nats" {
t.Errorf("wait-held %+v", b)
}
if !r.found["delivery.ace.nats.delivery-waits-outside"] {
t.Errorf("findings %v", r.found)
}
// the cause clears and the module runs healthy
r.conds, r.health, r.now = nil, nodeengineclient.HealthOK, r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.hub.unreachable","event":"cleared","change":"cleared","at":"2026-11-11T10:01:00Z","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
if len(r.events) != 2 || r.events[1] != meshwants.EventWaitEnded || r.bodies[1].How != outside.HowCleared || len(r.found) != 0 || len(r.p.w.read(buckets.Waits())) != 0 {
t.Errorf("events %v bodies %+v findings %v", r.events, r.bodies, r.found)
}
})
t.Run("a cause not listing the machine settles, then the failure is the delivery's", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "g14")
n := r.review(false, nil)
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 0 || !r.wait().Settling {
t.Fatalf("settle: health %s events %v wait %+v", healthOf(n), r.events, r.wait())
}
r.now = r.now.Add(91 * time.Second)
if n := r.review(false, nil); healthOf(n) != nodeengineclient.HealthFailed {
t.Errorf("past the settle the check still counts as failing: %s", healthOf(n))
}
})
t.Run("a cause turning into another keeps the wait and its clock; a rewritten naming a moved module fails at once", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
r.conds = nil
r.cond("machine.hub.network", "ace")
r.now = r.now.Add(5 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.network","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
last := len(r.bodies) - 1
if len(r.events) != 2 || r.events[last] != meshwants.EventWaitHeld || r.bodies[last].On != "network:hub" || !r.bodies[last].ClockFrom.Equal(*r.bodies[0].ClockFrom) {
t.Errorf("events %v bodies %+v", r.events, r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.ace.nats.rewritten")
n := r2.review(false, func(owner, node string) bool { return owner == "nats" })
if healthOf(n) == nodeengineclient.HealthFailed || !r2.wait().Settling {
t.Errorf("a rewritten of the moved module is its failure: %s %+v", healthOf(n), r2.wait())
}
})
t.Run("the keeper silent waits on conditions-unread, and its own module is judged with no read", func(t *testing.T) {
r := newOutsideRig(t)
r.unread = true
r.review(false, nil)
r.now = r.now.Add(90 * time.Second) // the settle has run with no answer
r.review(false, nil)
if len(r.bodies) != 1 || r.bodies[0].On != outside.OnUnread {
t.Fatalf("bodies %+v", r.bodies)
}
r2 := newOutsideRig(t)
r2.cond("machine.hub.unreachable", "ace")
if n := r2.review(true, nil); healthOf(n) != nodeengineclient.HealthFailed || len(r2.events) != 0 {
t.Errorf("the keeper's own failing check waits: %s %v", healthOf(n), r2.events)
}
})
t.Run("past the bound the operator is asked; Go on counts it healthy and Count as failed holds it, each said ended", func(t *testing.T) {
for how, want := range map[string]string{assign.OptionGoOn: nodeengineclient.HealthOK, assign.OptionFailed: nodeengineclient.HealthFailed} {
r := newOutsideRig(t)
r.cond("machine.hub.silent", "ace")
r.review(false, nil)
r.now = r.now.Add(31 * time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-changed", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
r.review(false, nil)
key, _ := asks.OutsideAskKey("ace", "nats", fpTest)
raw := r.p.w.value(buckets.Asks(), key)
if raw == nil {
t.Fatalf("%s: no ask at %s", how, key)
}
_, ask, err := assign.DecodeAsk(key, raw)
if err != nil || ask.About != "ace.nats."+strings.TrimPrefix(key[strings.LastIndex(key, ".")+1:], "") || len(ask.Options) != 2 {
t.Fatalf("ask %+v %v", ask, err)
}
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: how, Warrant: ask.ID + "@2026-11-11T10:32:00Z", Words: "the operator"})
r.p.w.put(buckets.Answers, key, v)
n := r.review(false, nil)
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != how || healthOf(n) != want {
t.Errorf("%s: events %v last %+v health %s", how, r.events, r.bodies[len(r.bodies)-1], healthOf(n))
}
if r.p.w.value(buckets.Asks(), key) != nil || len(r.found) != 0 {
t.Errorf("%s: the ask or the finding stands after the end", how)
}
// a check that goes on failing is not waited on again
r.now = r.now.Add(time.Minute)
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
<-r.woke
before := len(r.events)
r.review(false, nil)
if len(r.events) != before {
t.Errorf("%s: a decided check is waited on again", how)
}
}
})
t.Run("a newer want ends the wait replaced, and a restart judges it again", func(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}}
r.p.outsideReview(context.Background(), "nats", "src2:"+strings.Repeat("c", 64), []string{"ace"}, n, false, func(string, string) bool { return false })
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != outside.HowReplaced {
t.Errorf("events %v", r.events)
}
})
t.Run("its bounds are read at the start from the files the manifest declares (A-112)", func(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
env := map[string]string{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": write("wait", "45\n"), "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": write("settle", "7")}
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 45*time.Minute || settle != 7*time.Second {
t.Errorf("bound %v settle %v err %v, want 45m and 7s", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_WAIT_FILE"], env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = filepath.Join(dir, "missing"), write("empty", "")
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 0 || settle != 0 {
t.Errorf("bound %v settle %v err %v, want both defaults", bound, settle, err)
}
env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = write("bad", "soon")
if _, _, err := outsideSettings(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") {
t.Errorf("err %v, want one naming the file", err)
}
})
}
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
r := newOutsideRig(t)
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
for k, v := range r.p.w.read(buckets.Waits()) {
s.st.put(waitsState, k, v)
}
got, err := s.doWaits(map[string]any{"module": "nats"})
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
t.Errorf("wait = %+v", w)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("an unknown how is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on"}, "shanks.node-tools"); err == nil || len(acts) != 0 {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on", "why": "the hub is being rebuilt"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if len(acts) == 0 || !strings.Contains(acts[0], "hand-act") {
t.Errorf("the hand act is not recorded first: %v", acts)
}
keys, _ := s.st.keys(waitsState)
raw, _, _, _ := s.st.get(waitsState, keys[0])
if w, _ := outside.Decode(raw); w.Release != "go-on" || !strings.Contains(w.By, "the hub is being rebuilt") {
t.Errorf("wait = %+v", w)
}
// the process takes the release at its next pass: ended go-on, counted healthy
r.p.w.put(buckets.Waits(), keys[0], raw)
n := r.review(false, nil)
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
}
}
@@ -1,120 +0,0 @@
package main
import (
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
// The tools of the waits on an outside cause (design 47 G §706, §276's row *delivery on an outside cause*): `assigner_waits` reads
// each, the settle included, and `assigner_release` with `how` takes one. The tool writes the release into the wait's record and the
// assigner's process ends the wait at its next pass, saying wait-ended with that how.
// toolWaits is the tool that reads the outside waits.
const toolWaits = "assigner_waits"
// waitsState is the assigner's state of waits, as the runtime names a module's own state.
const waitsState = buckets.WaitsState
// waitSaid is one wait as assigner_waits answers it: its record and where it stands.
type waitSaid struct {
Key string `json:"key"`
outside.Wait
// Settling's end is in the record; Ends is when the bound runs from clock-from, empty for a settle.
BoundEnds string `json:"bound-ends,omitempty"`
}
// doWaits is the tool assigner_waits.
func (s *putTools) doWaits(args map[string]any) (any, error) {
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
sort.Strings(keys)
out := []any{}
for _, key := range keys {
raw, _, found, err := s.st.get(waitsState, key)
if err != nil || !found {
continue
}
w, err := outside.Decode(raw)
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
continue
}
said := waitSaid{Key: key, Wait: w}
if w.Standing() {
if d, err := time.ParseDuration(w.Bound); err == nil {
said.BoundEnds = w.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
}
out = append(out, said)
}
return map[string]any{"waits": out}, nil
}
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
// module, which the assigner's process takes at its next pass.
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
}
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
prefix := node + "." + module + "."
var key string
var w outside.Wait
for _, k := range keys {
if !strings.HasPrefix(k, prefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if c, err := outside.Decode(raw); err == nil && c.Decided == "" {
key, w = k, c
}
}
if key == "" {
return nil, fmt.Errorf("%s holds no wait of %s on an outside cause: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, how), Why: why,
Cause: "outside-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Release, w.By, w.Why = how, by+": "+why, why
data, err := outside.Encode(w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
}
// waitsTool is assigner_waits as the runtime serves it.
func (s *putTools) waitsTool() stdio.Tool {
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: a moved module's failing health check on a " +
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
"machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil), Run: s.doWaits}
}
@@ -1,6 +0,0 @@
package main
import plc "mesh-delivery-assigner/internal/assignment"
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
func init() { plc.SetModule("mesh-delivery-assigner") }
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,384 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
// The wants follow a passed pipeline (design 47 H §750): the runner says each pipeline's end once, per source revision, as
// pipeline-ended on `pipeline-ended.<branch>.<source>`, naming per mover its fingerprint, its released version and its tier
// (§746). The assigner keeps, in its state `written` (G §352), one key per module and watched branch: the newest released
// version of the module on that branch whose pipeline passed, with the pipeline and the mover's rank in it. A newer version
// replaces it by compare-and-set; an older one, the same one again, a failed pipeline and a run on request write nothing.
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
const (
writtenState = "written"
writtenTool = "assigner_written"
)
// pipelineEndedSubject is the runner's event the assigner consumes.
const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
// End is one value of the state written: the newest version of a module released on a watched branch whose pipeline passed.
type End struct {
Pipeline string `json:"pipeline"`
Source int64 `json:"source"`
Fp string `json:"fp"`
Version string `json:"version"`
// Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750).
Rank int64 `json:"rank"`
}
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
func endKey(module, branch string) string {
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
}
// moverPassed says whether no required step of a mover stands failed, unasked or unended: a required step ending "nothing to
// run" counts as passed, as a bypassed one does (design 47 H §746, H-A-3); a failed optional step holds nothing (G §30).
func moverPassed(m mpr.MoverEnded) bool {
for _, s := range m.Steps {
if s.Status != mpr.StepResultStatusRequired {
continue
}
switch s.Outcome {
case mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeCouldNotRun, mpr.StepResultOutcomeDeclined, mpr.StepResultOutcomeNotAsked,
mpr.StepResultOutcomeCancelled, mpr.StepResultOutcomeSuperseded, mpr.StepResultOutcomeDidNotBuild,
mpr.StepResultOutcomeRunning, mpr.StepResultOutcomeWaiting, mpr.StepResultOutcomeWaitingForABuildNode:
return false
}
}
return true
}
// endStore is the kept state written: the bus bucket, or a test's.
type endStore interface {
// get is a key's value and revision; none for a key not there.
get(key string) (value []byte, revision uint64, err error)
// swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not.
swap(key string, value []byte, revision uint64) error
// keys are the keys that stand, and drop deletes one.
keys() ([]string, error)
drop(key string) error
}
var errSwapped = errors.New("the key changed")
// kvEnds is endStore over a bus bucket.
type kvEnds struct {
ctx context.Context
kv jetstream.KeyValue
}
func (k kvEnds) get(key string) ([]byte, uint64, error) {
e, err := k.kv.Get(k.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (k kvEnds) swap(key string, value []byte, revision uint64) error {
var err error
if revision == 0 {
_, err = k.kv.Create(k.ctx, key, value)
} else {
_, err = k.kv.Update(k.ctx, key, value, revision)
}
if errors.Is(err, jetstream.ErrKeyExists) || (err != nil && isWrongSequence(err)) {
return errSwapped
}
return err
}
func (k kvEnds) keys() ([]string, error) {
ks, err := k.kv.Keys(k.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return ks, err
}
func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) }
func isWrongSequence(err error) bool {
var ae *jetstream.APIError
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
}
// pipelineEnds follows the runner's ends; a nil *pipelineEnds keeps none, and no want is written from it.
type pipelineEnds struct {
st endStore
wake func()
logf func(string, ...any)
// on says the first registration was heard (design 47 G §353), said once in the journal.
on atomic.Bool
mu sync.Mutex
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
// 47 G §14).
firsts map[string]string
// runEnded is told a run on request's pipeline-ended, its group `run-<id>`, to drop the run's candidate (A-235, H-A-58); nil
// drops nothing.
runEnded func(group string) error
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
left atomic.Pointer[func(pipeline string) error]
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
trunk func() string
// now is the clock of the tier progress (tiers.go); nil is time.Now.
now func() time.Time
}
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
}
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). It says whether any key changed.
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
return false, nil
}
changed := false
var kept []tierMover
versioned := map[string]bool{}
defer func() {
if err := e.startTiers(p.Pipeline, p.Source, kept, versioned); err != nil {
e.logf("the tier progress of %s is not written: %v", p.Pipeline, err)
}
}()
for _, m := range p.Movers {
if m.Fingerprint == "" || !moverPassed(m) {
continue
}
if m.Version == "" {
// The runner writes a mover's version at its release's version-released, and says this end again once it does
// (H-A-270 (1), (2)): the skip is said, never silent.
e.logf("%s: %s passed with no version yet: taken again when its version is written", p.Pipeline, m.Module)
// It holds its tier in the progress until its version is written (H-A-277 (1)).
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1), NoVersion: true})
continue
}
versioned[m.Module] = true
key := endKey(m.Module, p.Branch)
for {
raw, rev, err := e.st.get(key)
if err != nil {
return changed, err
}
if raw != nil {
var kept End
if err := json.Unmarshal(raw, &kept); err == nil && compareVersion(m.Version, kept.Version) <= 0 {
break // an older version, or this one again
}
}
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
if err != nil {
return changed, err
}
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else if err != nil {
return changed, err
}
changed = true
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1)})
break
}
}
return changed, nil
}
// endOf is the end kept for module on branch: the newest version released there whose pipeline passed.
func (e *pipelineEnds) endOf(module, branch string) (End, bool) {
if e == nil {
return End{}, false
}
raw, _, err := e.st.get(endKey(module, branch))
var kept End
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp == "" {
return End{}, false
}
return kept, true
}
// target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline
// passed, with the pipeline and the mover's rank; false when none is kept, or its pipeline's tier progress has not reached its
// rank (§751).
func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) {
end, ok := e.endOf(module, e.branchOf(module, branch))
if !ok || !e.tierWritten(end) {
return assign.Target{}, false
}
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true
}
// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the
// run's candidate assignment graph (A-235, H-A-58); a run delivers nothing (design 47 G §29, §64). A pipeline ended superseded or
// cancelled has its composition steps leave the runs they read (G §723).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if err != nil {
e.logf("%s is not taken: %v", m.Subject(), err)
_ = m.Ack()
return
}
if left := e.left.Load(); left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
if p.Run != "" {
if e.runEnded != nil {
if err := e.runEnded(runGroupPrefix + p.Run); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
changed, err := e.take(p)
if err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
if changed {
e.wake()
}
_ = m.Ack()
}
// hearEnds registers pipeline-ended on the assigner's hub.
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
}
return nil
}
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
func (e *pipelineEnds) branchOf(module, placed string) string {
if placed != "" {
return placed
}
e.mu.Lock()
defer e.mu.Unlock()
if b := e.firsts[module]; b != "" {
return b
}
if e.trunk != nil {
return e.trunk()
}
return ""
}
// writtenTools is the tool assigner_written: every key of the state written, with its end.
func writtenTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the newest version released whose pipeline passed, with its pipeline and rank; per pipeline whose wants are written tier by tier, its tier progress (tier, since, movers); with why, why each want is what it is (as assigner_why).",
Input: schema(map[string]string{"why": "true: why each want is what it is", "module": "with why: a module's name", "node": "with why: a node's name"}, nil, "why"),
Run: func(args map[string]any) (any, error) {
if args["why"] == true || args["why"] == "true" {
return whyTools(st)[0].Run(args) // a want's why (why.go), until the manifest naming assigner_why is taken in
}
ks, err := st.keys(writtenState)
if err != nil {
return nil, err
}
sort.Strings(ks)
out := map[string]any{}
for _, k := range ks {
if strings.HasPrefix(k, whyPrefix) {
continue // a want's why (why.go), read by assigner_why
}
raw, _, ok, err := st.get(writtenState, k)
if err != nil || !ok {
continue
}
if isTierKey(k) { // a pipeline's tier progress (tiers.go)
var t tierProgress
if json.Unmarshal(raw, &t) == nil {
out[k] = t
}
continue
}
var e End
if json.Unmarshal(raw, &e) == nil {
out[k] = e
}
}
return out, nil
}}}
}
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first
// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone
// is a registration removed: all its ends go.
func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
e.mu.Lock()
if gone || len(reg.Branches) == 0 {
delete(e.firsts, reg.Module)
} else {
e.firsts[reg.Module] = reg.Branches[0]
}
e.mu.Unlock()
kept, err := e.st.keys()
if err != nil {
e.logf("the ends kept cannot be listed: %v", err)
}
prefix := keys.EscapeName(reg.Module) + "."
for _, k := range kept {
rest, ok := strings.CutPrefix(k, prefix)
if !ok || isTierKey(k) || strings.HasPrefix(k, whyPrefix) {
continue
}
branch, err := keys.UnescapeName(rest)
if err == nil && !gone && slices.Contains(reg.Branches, branch) {
continue
}
if err := e.st.drop(k); err != nil {
e.logf("the end %s is not deleted: %v", k, err)
}
}
if e.onBranches != nil && !gone {
e.onBranches(reg.Branches)
}
if !e.on.Swap(true) {
e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module)
}
e.wake()
}
@@ -1,292 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// memEnds is endStore over a map, with revisions.
type memEnds struct {
vals map[string][]byte
revs map[string]uint64
}
func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil }
func (m *memEnds) keys() (out []string, _ error) {
for k := range m.vals {
out = append(out, k)
}
return out, nil
}
func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil }
func (m *memEnds) swap(key string, v []byte, rev uint64) error {
if m.revs[key] != rev {
return errSwapped
}
m.vals[key], m.revs[key] = v, rev+1
return nil
}
// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back.
type heardMsg struct {
jetstream.Msg
data []byte
subject string
acked, naked bool
}
func (m *heardMsg) Data() []byte { return m.data }
func (m *heardMsg) Subject() string { return m.subject }
func (m *heardMsg) Ack() error { m.acked = true; return nil }
func (m *heardMsg) Nak() error { m.naked = true; return nil }
// mover is one mover at its pipeline's end, its required unit step ending outcome.
func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnded {
return mpr.MoverEnded{Module: module, Fingerprint: fp, Version: version, Tier: tier,
Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}}
}
// ended is the pipeline-ended of the take-in of source on branch, its movers given.
func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded {
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers,
Pipeline: fmt.Sprintf("%s.%d", branch, source)}
}
// A6/AC1 to AC3 (design 47 H §750, H-A-3): a passed pipeline keeps each mover's released version as the newest of its module on
// the branch, with the pipeline and the mover's tier; a failed pipeline, a mover whose required step failed, one that released
// no version, an older version, the same one again and a run on request keep nothing.
func TestThePassedVersionIsKept(t *testing.T) {
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
ok, failed, nothing := mpr.StepResultOutcomePassed, mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeNothingToRun
for _, c := range []struct {
name string
ends []mpr.PipelineEnded
want *End // nil: none kept for web on main
}{
{"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))},
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
{"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))},
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}},
{"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok),
mover("db", "fp9", "0.3.0", 1, failed))}, nil},
{"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil},
{"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil},
{"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)),
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
p.Run = "r1"
return p
}()}, nil},
} {
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
for _, p := range c.ends {
if _, err := e.take(p); err != nil {
t.Fatalf("%s: %v", c.name, err)
}
}
got, kept := e.endOf("web", "main")
switch {
case c.want == nil && kept:
t.Errorf("%s: kept %+v", c.name, got)
case c.want != nil && (!kept || got != *c.want):
t.Errorf("%s: kept %+v (%v), want %+v", c.name, got, kept, *c.want)
}
}
// The same end heard again changes nothing and wakes nothing; a run on request's end drops its candidate.
woken := 0
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
var dropped []string
e.runEnded = func(group string) error { dropped = append(dropped, group); return nil }
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
for range 2 {
raw, _ := json.Marshal(p)
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"}
e.hearPipeline(m)
if !m.acked {
t.Error("the end was not acknowledged")
}
}
run := p
run.Run = "r1"
raw, _ := json.Marshal(run)
e.hearPipeline(&heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"})
if woken != 1 || !slices.Equal(dropped, []string{runGroupPrefix + "r1"}) {
t.Errorf("woken %d (want 1), dropped %v (want the run's candidate)", woken, dropped)
}
}
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
// says so in the journal; the same end said again naming the version writes it.
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
for _, passed := range []bool{false, true} {
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
mdigest := strings.Repeat("e", 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web", Requires: []string{"database"}}}
placeOn(t, p.w, "g14", "web")
placeOn(t, p.w, "ace", "web")
var journal []string
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
p.ends.trunk = p.w.trunkName
if passed {
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
p.wants(context.Background())
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
t.Errorf("an end naming no version wrote %d wants and said %q", len(wantsKV.values), journal)
}
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 2, mpr.StepResultOutcomePassed)))
}
p.wants(context.Background())
for _, node := range []string{"g14", "ace"} {
key, _ := mwc.WantKey(node, "web")
raw, got := wantsKV.values[key]
if got != passed {
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
continue
}
if !passed {
continue
}
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
t.Errorf("the want on %s = %+v (%v)", node, w, err)
}
}
}
}
// Design 47 G §353, §352: a registration's first branch is a placement's default, and a branch it no longer watches loses its
// end; a registration gone loses them all.
func TestARegistrationNamesTheDefaultBranch(t *testing.T) {
st := newMemEnds()
woken := 0
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
e.trunk = func() string { return trunkBranch }
pass := mpr.PipelineEndedOutcomePassed
_, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed)))
_, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed)))
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
endsKept := func() (n int) { // a pipeline's tier progress is not a module's end
for k := range st.vals {
if !isTierKey(k) {
n++
}
}
return n
}
if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", ""))
}
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" {
t.Errorf("the default branch's target = %+v %v", tg, ok)
}
e.registered(mpcat.Registration{Module: "web"}, true)
if endsKept() != 0 {
t.Errorf("a registration gone left %d ends", endsKept())
}
}
// Design 47 G §14, §184 (test 68): a node's want follows the branch its placement names, so a version passed on `beta` moves only
// the want of the node whose placement chose `beta`.
func TestAVersionOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
values, err := mmac.AssignmentsValues(mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}})
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
}
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp("a"), Manifest: strings.Repeat("e", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, "main", "web", mod)
passed := func(branch, f, version string, source int64) {
_, _ = p.ends.take(ended(branch, source, mpr.PipelineEndedOutcomePassed, mover("web", f, version, 1, mpr.StepResultOutcomePassed)))
}
fpWant := func(node string) string {
key, _ := mwc.WantKey(node, "web")
w, err := mwc.DecodeWant(wantsKV.values[key])
if err != nil {
t.Fatalf("the want of web on %s: %v", node, err)
}
return w.Fp
}
passed("main", fp("a"), "0.1.1", 5)
passed("beta", fp("b"), "0.2.1", 5)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
}
passed("beta", fp("d"), "0.2.2", 6)
p.wants(context.Background())
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
t.Errorf("a version on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
}
}
// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node
// the rules read, and a registration's branches are each followed.
func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
w := newWorld()
woken := 0
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
entry := func(fp string) json.RawMessage {
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
return raw
}
h.changed("main", "web", entry("src2:1"), false)
h.changed("beta", "web", entry("src2:2"), false)
if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 {
t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken)
}
v := w.trunkOf("beta")["web"]
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
}
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
t.Errorf("graph node %+v", n)
}
h.changed("beta", "web", nil, true)
if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" {
t.Error("a module taken out of beta's half leaves main's")
}
}
@@ -1,204 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
func orderFP(c string) string {
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
}
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
type orderMod struct {
name string
run, trunk string
forms map[string]int
holds []string
}
type orderFixture struct {
t *testing.T
p *passer
wants *memKV
events []string
bodies []meshwants.Wait
answers chan struct{}
}
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
t.Helper()
ctx := context.Background()
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
p := f.p
p.logf = t.Logf
p.say = func(oc.Condition) error { return nil }
p.publishWants = func(event string, body any, id string) error {
f.events = append(f.events, event)
f.bodies = append(f.bodies, body.(meshwants.Wait))
return nil
}
digest := "sha256:" + strings.Repeat("c", 64)
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
pl := mma.Assignments{Node: "ace"}
forms := map[string]int{} // by fingerprint: each module's own
modules := []string{}
for _, m := range mods {
modules = append(modules, m.name)
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
for _, seat := range m.holds {
scope := "node"
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
scope = "mesh"
}
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
}
for fp, form := range m.forms {
forms[fp] = form
}
mdigest := fmt.Sprintf("%064x", len(modules))
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, m.name, mod)
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
endDone(t, p.w, m.name, m.trunk, mdigest)
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
f.runs(m.name, m.run)
// The trunk's build is released with a passed pipeline (design 47 H §750).
_, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover(m.name, m.trunk, "0.1.1", 1, mpr.StepResultOutcomePassed)))
wkey, _ := mwc.WantKey("ace", m.name)
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
p.w.put(buckets.Wants(), wkey, old)
f.wants.values[wkey] = old
}
composes(t, p.w, "ace", digest)
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket(), key, raw)
}
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
objects := memObjects{}
for i, at := range history {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, module := range modules {
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
}
placed := mma.Assignments{Node: "ace"}
for _, module := range modules {
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]mma.Assignments{"ace": placed}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
if form := forms[in.Fingerprint]; form > 0 {
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
return f
}
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
func (f *orderFixture) runs(module, fp string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
f.p.w.put(buckets.Have, "ace."+module, have)
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
}
// pass makes a pass, and after it every catalog entry it asked is answered.
func (f *orderFixture) pass() {
f.t.Helper()
f.p.wants(context.Background())
asked := 0
for _, row := range f.p.entries.Rows() {
if row.State == meshcatalogclient.EntryAsked {
asked++
}
}
for range asked {
select {
case <-f.answers:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
f.t.Fatal("an entry asked was not answered")
}
}
}
// wantAt is the fingerprint module is wanted at on ace.
func (f *orderFixture) wantAt(module string) string {
f.t.Helper()
key, _ := mwc.WantKey("ace", module)
w, err := mwc.DecodeWant(f.wants.values[key])
if err != nil {
f.t.Fatal(err)
}
return w.Fp
}
func (f *orderFixture) waits() []string {
var out []string
for key := range f.p.w.read(buckets.Waits()) {
out = append(out, key)
}
return out
}
// A3/AC1 (design 47 H §771, §764): given a module whose kept data's form changes while the new versions of the node-backup holder and
// of the bus module do not run, when the assigner passes, then the module is wanted at its new version, no `order.` key is kept and no
// wait is said: the order on a machine is its node-engine's apply order, by rank.
func TestNoWantWaitsForTheOrderOfADelivery(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
})
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 0 {
t.Fatalf("web at %s, waits %v, events %v: want web at its new version, no wait kept or said", got, f.waits(), f.events)
}
}
@@ -5,23 +5,15 @@ import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"mesh-delivery-assigner/internal/assign"
)
@@ -41,12 +33,6 @@ const PutEvent = "put"
type caller interface {
Call(key string, body any) (json.RawMessage, error)
CallOn(node, key string, body any) (json.RawMessage, error)
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
CallKept(key string, body any) (json.RawMessage, error)
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
CallFile(key string, body any) (json.RawMessage, error)
// CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775).
CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
@@ -62,40 +48,40 @@ type putActs struct {
snapNo map[string]string
warranted map[string]bool
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
rec meshhandactsclient.Recorder
rec handacts.Recorder
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
snapshot func(ctx context.Context, node, module string) (string, error)
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{rec: meshhandactsclient.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p puts.Put) string {
func identity(key string, p delivery.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]puts.Put {
out := map[string]map[string]puts.Put{}
for key, value := range p.w.read(buckets.Puts()) {
node, module, err := puts.ParseKey(key)
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
}
put, err := puts.Decode(value)
put, err := delivery.DecodePut(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]puts.Put{}
out[module] = map[string]delivery.Put{}
}
out[module][node] = put
}
@@ -108,12 +94,12 @@ func (p *passer) puts(ctx context.Context) {
return
}
all := p.putsByModule()
defer func() { p.putWaits(ctx, p.putsByModule()) }()
if len(all) == 0 {
return
}
assigned := p.w.assigned()
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
@@ -123,26 +109,34 @@ func (p *passer) puts(ctx context.Context) {
for _, module := range modules {
var manifest catalogue.Manifest
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
if got := p.w.manifest(mod.Manifest); got.err == "" {
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
manifest = got.manifest
}
}
for node, put := range all[module] {
if puts.Ended(put.Step) {
if delivery.PutEnded(put.Step) {
continue
}
key, _ := puts.Key(node, module)
key, _ := delivery.PutKey(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, puts.Refused, "the bus's want is not written: "+err.Error())}
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) &&
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
@@ -151,67 +145,71 @@ func (p *passer) puts(ctx context.Context) {
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted {
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !mwc.SameWant(normalWant(before), normalWant(want)) {
p.putWant(ctx, node, module, want)
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, meshdeliveryclient.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w mwc.Want) mwc.Want {
func normalWant(w delivery.Want) delivery.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].Fp2}
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
w.Build, w.BuildWords = e.Outcome, e.Error
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
if c, ok := chosen[module]; ok {
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
} else if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := asks.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey)); err == nil && a.Asking == assign.PutAsking(put) {
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
case p.answeredWith(akey, assign.OptionNo):
w.Answer = assign.OptionNo
default:
expires := assign.ExpiresAt(ask)
w.AskExpired = !expires.IsZero() && w.Now.After(expires)
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
}
}
}
}
if r, ok := p.w.runningOf(node, module); ok {
w.Running = &r
}
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
w.Have = &h
}
}
@@ -222,15 +220,15 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
var out *assign.ComposedDigest
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := grantsstate.PartGrantKey(module, fp); err == nil {
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
@@ -240,17 +238,17 @@ func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, b
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == puts.Person && next.Put.Ask != "" && w.Composed != nil {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey))
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
}
}
}
@@ -259,29 +257,29 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
}
}
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
func (p *passer) performs(key string, act occ.Act) error {
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks(), key))
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
func (p *passer) performs(key string, act asks.Act) error {
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
for _, o := range ask.Options {
for _, o := range a.Ask.Options {
if o.ID == assign.OptionYes {
return occ.Performs(o, act)
return o.Performs(act)
}
}
return fmt.Errorf("the ask %s offers no yes", key)
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b puts.Put) bool {
func samePut(a, b delivery.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put) bool {
kv := p.kvs[buckets.Puts()]
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
if kv == nil {
return false
}
@@ -290,11 +288,11 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := puts.Decode(e.Value())
now, err := delivery.DecodePut(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := puts.Encode(next)
data, err := delivery.EncodePut(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
@@ -305,7 +303,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
}
return false
}
p.w.put(buckets.Puts(), key, data)
p.w.put(delivery.AssignerPutsBucket, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
@@ -401,7 +399,7 @@ func (a *putActs) warrant(id, ask string) {
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
@@ -430,8 +428,8 @@ type putSaid struct {
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p puts.Put) {
node, module, err := puts.ParseKey(key)
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
if err != nil || a.bus == nil {
return
}
@@ -1,116 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its
// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint
// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release,
// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none.
// putWaitBound is the bound of a put without hold's wait.
const putWaitBound = 30 * time.Minute
// Wait-ended's how of a put's wait.
const (
putWaitRan = "ran"
putWaitReleased = "released"
putWaitSuperseded = "superseded"
)
// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run.
func putWaits(p puts.Put) bool {
return !p.Hold && p.Step == puts.Wanted
}
// putWaitBody is the wait of the put p of module on node.
func putWaitBody(node, module string, p puts.Put) meshwants.Wait {
since := p.StandsSince().UTC()
return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module(), Node: node, Module: module, Fingerprint: p.FP,
Since: since, ClockFrom: &since, Bound: putWaitBound.String(),
Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)}
}
// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204,
// A-211): its identity and the put as it was then.
type putWaiting struct {
ID string `json:"id"`
Put puts.Put `json:"put"`
}
// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left.
func (p *passer) saidPutWaits() map[string]putWaiting {
out := map[string]putWaiting{}
for key, value := range p.w.read(buckets.PutWaits()) {
var w putWaiting
if err := json.Unmarshal(value, &w); err != nil || w.ID == "" {
p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits(), key, err)
continue
}
out[key] = w
}
return out
}
// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended
// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said;
// one not said, or not kept, is said at the next pass. Without the state no put's wait is said.
func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) {
if p.kvs[buckets.PutWaits()] == nil {
return
}
waiting := p.saidPutWaits()
for key, was := range waiting {
node, module, err := puts.ParseKey(key)
if err != nil {
continue
}
cur, there := all[module][node]
same := there && identity(key, cur) == was.ID
if same && putWaits(cur) {
continue
}
how := putWaitSuperseded
if same && cur.Step == puts.Done {
how = putWaitRan
} else if same && cur.Step == puts.Released {
how = putWaitReleased
}
body := putWaitBody(node, module, was.Put)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil {
p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
p.remove(ctx, buckets.PutWaits(), key)
delete(waiting, key)
}
for module, byNode := range all {
for node, put := range byNode {
key, err := puts.Key(node, module)
if err != nil || !putWaits(put) {
continue
}
id := identity(key, put)
if w, said := waiting[key]; said && w.ID == id {
continue
}
if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil {
p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
kept := putWaiting{ID: id, Put: put}
p.put(ctx, buckets.PutWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
}
}
}
@@ -1,92 +0,0 @@
package main
import (
"context"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as
// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is
// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said
// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one
// that ran while the assigner was stopped gets its wait-ended at the next start.
func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) {
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
kept := &memKV{values: map[string][]byte{}}
var events []string
var bodies []meshwants.Wait
start := func() *passer { // a process's start: a fresh world that hands what the state holds
p := newPasser(kvSet{buckets.PutWaits(): kept}, at)
for k, v := range kept.values {
p.w.put(buckets.PutWaits(), k, v)
}
p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {})
p.publishWants = func(event string, body any, id string) error {
events, bodies = append(events, event), append(bodies, body.(meshwants.Wait))
return nil
}
return p
}
ctx := context.Background()
asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked")
wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed")
held := wanted
held.Hold = true
p := start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil {
t.Fatalf("a put that waits said %v, kept %q", events, kept.values)
}
events, bodies = nil, nil
// At the next start the put still waits: it was said before, nothing is said of it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 0 {
t.Fatalf("a put waiting at the start was said again: %v", events)
}
// It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}})
if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut ||
kept.values["ace.nats"] != nil {
t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values)
}
events, bodies = nil, nil
// A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing.
again := wanted
again.At = at.Add(time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("a put's written want said %v", events)
}
b := bodies[0]
if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil ||
!b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" {
t.Errorf("the put's wait is %+v", b)
}
// Said once.
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 {
t.Errorf("a put's wait was said again: %v", events)
}
// Put again under its key: the first wait ends superseded, the new one is held.
newer := asked
newer.At = at.Add(2 * time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}})
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest {
t.Errorf("a put asked again said %v %+v", events, bodies)
}
// The answer of assigner_puts.
if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" {
t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound)
}
if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" {
t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound)
}
}
@@ -1,87 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
type releaseHeard struct {
Version string
Parts map[string]string
Source int64
At time.Time
}
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
func presenceOf(data []byte) (string, time.Time, error) {
var e wd.MachineAwayEvent // machine-back has the same fields
if err := json.Unmarshal(data, &e); err != nil {
return "", time.Time{}, err
}
if e.Machine == "" {
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
}
return e.Machine, e.At, nil
}
// hearReleases reads whole, from their first message, then follows, the catalog seat's version-released and the watchman's
// machine-away and machine-back (mesh-sdk process.FromFirst, A-233), never their state. A stream not heard is said as the assigner's finding, and what it carries is not known until
// the assigner starts again.
func hearReleases(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
for _, s := range []struct {
stream, words string
take func([]byte) error
}{
{meshcatalog.StreamVersionReleased, "no want names a released version", w.released.put},
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, true, at)
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", m)
}
return err
}},
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, false, at)
logf("%s is back (machine-back): a composition step composes it again", m)
}
return err
}},
} {
take := s.take
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
if err := take(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf, process.FromFirst())
if err != nil {
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
continue
}
nh.Heard(s.stream)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
}
@@ -1,177 +0,0 @@
package main
import (
"context"
"strconv"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
var revertReasons = []string{"restore point missing", "restore point damaged"}
// revertReason is the reason of a have's words when they say a held revert, or "".
func revertReason(words string) string {
for _, r := range revertReasons {
if strings.HasPrefix(words, r) {
return r
}
}
return ""
}
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
// that holds a revert, which backups leaves to it.
func (p *passer) reverts(ctx context.Context) map[string]bool {
taken := map[string]bool{}
wanted := p.w.wantsOf()
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := nodeengineclient.DecodeHave(value)
if err != nil {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := asks.RevertAskKey(node, module, want.Fp)
if err != nil {
continue
}
reason := revertReason(have.Detail)
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
if have.Fp == want.Fp && p.w.has(buckets.Asks(), akey) {
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
}
continue
}
taken[node+"."+module] = true
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
if !ok {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
if there != nil {
now := p.clock()
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
continue
}
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
continue // not answered, or answered hold: the hold stands and nothing is written
}
words := assign.RevertAnswerWords(answer.Option, current.Asking)
if want.Revert != mwc.RevertLive || want.Words != words {
want.Revert, want.Words = mwc.RevertLive, words
p.putWant(ctx, node, module, want)
}
}
return taken
}
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
if p.graph == nil || p.entries == nil {
return "", "", false
}
run, heard := p.w.runningOf(node, module)
if !heard || run.Stopped || run.Fingerprint == "" {
return "", "", false
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
return "", "", false
}
if !reverted {
return "", "", false
}
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
if !known {
return "", "", false // asked: its answer wakes a pass
}
var names, forms []string
for _, c := range changes {
f := 0
if c.Applied != nil {
f = c.Applied.Form
}
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
}
if len(names) == 0 {
return "", "", false
}
return strings.Join(names, ", "), strings.Join(forms, ", "), true
}
// revertFinding writes the finding revert-held, or clears it, once per process each way.
func (p *passer) revertFinding(o assign.Observed, raised bool) {
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
said := key + ".raised"
if !raised {
said = key + ".cleared"
}
if p.said[said] {
return
}
var err error
switch {
case raised && p.findings.Put != nil:
err = p.findings.Put(o.Finding(p.clock()))
case !raised && p.findings.Delete != nil:
err = p.findings.Delete(key)
default:
return
}
if err != nil {
p.logf("the finding %s is not written: %v", key, err)
return
}
p.said[said] = true
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
}
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
if p.entries == nil {
return nil, false
}
running, rs := p.entries.Need(ctx, runFp)
applied, as := p.entries.Need(ctx, wantFp)
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
return nil, false
}
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
}
changes, err := assigngraph.FormChanges(running, applied)
if err != nil {
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
return nil, false
}
return changes, true
}
@@ -1,154 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, at)
p.logf = t.Logf
findings := map[string][]byte{}
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
objects := memObjects{}
for i, fp := range []string{fpOld, fpTest} {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
entry := func(fp string, form int) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
}
answered := make(chan struct{}, 4)
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
wkey, _ := mwc.WantKey("ace", "web")
want := mwc.Want{Fp: fpOld, Self: true}
raw, _ := mwc.EncodeWant(want)
p.w.put(buckets.Wants(), wkey, raw)
hkey, _ := nodeengineclient.HaveKey("ace", "web")
have := func(fp, phase, detail string) {
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
}
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
pass := func() { p.backups(ctx, p.reverts(ctx)) }
pass() // the entries are asked; nothing is asked of the operator until they answer
for range 2 {
select {
case <-answered:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
}
}
pass()
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
if !p.w.has(buckets.Asks(), rkey) || p.w.has(buckets.Asks(), bkey) {
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
}
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks())[rkey])
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
t.Fatalf("the revert ask = %+v, %v", ask, err)
}
fkey := "delivery.ace.web.revert-held"
if _, raised := findings[fkey]; !raised {
t.Fatalf("findings %v: want %s", findings, fkey)
}
answer := func(option string) {
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
p.w.put(buckets.Answers, rkey, v)
}
puts := wantsKV.puts
answer(assign.OptionRevertHold)
pass()
if wantsKV.puts != puts {
t.Fatal("an answer hold wrote a want")
}
answer(assign.OptionRevertLive)
pass()
got, err := mwc.DecodeWant(p.w.read(buckets.Wants())[wkey])
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
t.Fatalf("the want after live = %+v, %v", got, err)
}
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
pass()
if _, raised := findings[fkey]; raised {
t.Fatal("the finding stands while the reverted build runs")
}
// A newer want replaces the held revert: the ask goes and the finding with it.
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
pass()
if _, raised := findings[fkey]; !raised {
t.Fatal("the finding is not raised again for a hold again")
}
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
p.w.put(buckets.Wants(), wkey, raw)
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
}
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
for _, withWait := range []bool{true, false} {
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
if withWait {
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
}
raw, err := nodeengineclient.EncodeHave(h)
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
pass()
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
}
}
}
@@ -1,184 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// The assignment graph revisions are removed when they are no longer current (design 47 H §764, H-A-86): the gate's pins of
// them went with its pinned, freed, heads-replaced and pins-stated (H-A-48), and with them the assigner's state pins and its
// tools assigner_pins and assigner_unpin. A revision that is not current (the newest the assigner wrote) is removed, its object
// and the entry objects no kept revision or candidate lists and no want or running build names (G §179), when a revision ceases
// to be current and once at the start. Never on a timer.
// revisionObjects is the object bucket of the revisions as the remover removes from it.
type revisionObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// revisionRemover removes the revisions that are not current.
type revisionRemover struct {
ctx context.Context
objects revisionObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
logf func(string, ...any)
}
// judge removes every revision that is not current; a removal that fails is said and made at the next judging.
func (k *revisionRemover) judge() {
if err := k.remove(); err != nil {
k.logf("the revisions that are not current are not removed: %v", err)
}
}
// trunkRevision is the revision an object of a trunk revision names, and whether it names one.
func trunkRevision(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or
// running build names (design 47 G §179).
func (k *revisionRemover) remove() error {
names, described, err := k.listing()
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := trunkRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
if r == current {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its run's pipeline-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := k.entryOf(n, digest, described)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions not current are removed", removed)
return nil
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
// listing is every object's name in one read and, where the bucket says them, each entry object's module and fingerprint by name
// (design 47 H-A-259); nil described where the bucket says none.
func (k *revisionRemover) listing() ([]string, map[string]assigngraph.Described, error) {
d, ok := k.objects.(assigngraph.Describer)
if !ok {
names, err := k.objects.Names(k.ctx)
return names, nil, err
}
listed, err := d.Described(k.ctx)
if err != nil {
return nil, nil, err
}
names := make([]string, 0, len(listed))
described := make(map[string]assigngraph.Described, len(listed))
for _, l := range listed {
names = append(names, l.Name)
described[l.Name] = l
}
return names, described, nil
}
// entryOf is the module and fingerprint of the entry object name: as the listing says them, with no read; an entry put before the
// listing said them is read once and put again with them, so no later judging reads it (design 47 H-A-259).
func (k *revisionRemover) entryOf(name, digest string, described map[string]assigngraph.Described) (meshwants.AssignmentEntry, error) {
if l, ok := described[name]; ok && l.Module != "" && l.Fp != "" {
return meshwants.AssignmentEntry{Module: l.Module, Fp: l.Fp}, nil
}
data, err := k.objects.Get(k.ctx, name)
if err != nil {
return meshwants.AssignmentEntry{}, fmt.Errorf("the entry %s: %w", digest, err)
}
var e meshwants.AssignmentEntry
if err := json.Unmarshal(data, &e); err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
if described != nil {
if err := assigngraph.PutEntry(k.ctx, k.objects, name, data, e); err != nil {
k.logf("the entry %s is not put again with its module and fingerprint, so the next judging reads it again: %v", digest, err)
}
}
return e, nil
}
@@ -1,110 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// describedObjects is memObjects whose listing says each entry's module and fingerprint (assigngraph.Describer), counting the
// entry objects read.
type describedObjects struct {
memObjects
meta map[string][2]string
reads int
}
func (d *describedObjects) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") {
d.reads++
}
return d.memObjects.Get(ctx, name)
}
func (d *describedObjects) PutDescribed(_ context.Context, name string, data []byte, module, fp string) error {
d.memObjects[name], d.meta[name] = data, [2]string{module, fp}
return nil
}
func (d *describedObjects) Described(context.Context) ([]assigngraph.Described, error) {
var out []assigngraph.Described
for n := range d.memObjects {
out = append(out, assigngraph.Described{Name: n, Module: d.meta[n][0], Fp: d.meta[n][1]})
}
return out, nil
}
// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the
// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists,
// a running build's entry and a run's kept candidate with its entries stay.
// H-A-259: where the listing says each entry's module and fingerprint, the judging reads no entry object; an entry put before it
// said them is read once and put again with them.
func TestARevisionNotCurrentIsRemoved(t *testing.T) {
for _, listed := range []string{"plain", "described", "described-later"} {
objects := memObjects{}
described := &describedObjects{memObjects: objects, meta: map[string][2]string{}}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
if listed == "described" {
described.meta[assigngraph.EntryObject(d)] = [2]string{module, fp}
}
return d
}
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
var o revisionObjects = objects
if listed != "plain" {
o = described
}
k := &revisionRemover{ctx: context.Background(), objects: o, logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
k.judge()
has := func(name string) bool { _, ok := objects[name]; return ok }
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
if has(name) != want {
t.Errorf("%s: %s kept = %v, want %v", listed, name, has(name), want)
}
}
switch listed {
case "described":
if described.reads != 0 {
t.Errorf("described: the judging read %d entry objects, want none", described.reads)
}
case "described-later":
if described.meta[assigngraph.EntryObject(running)] != [2]string{"c", "fp-c1"} {
t.Errorf("described-later: the entry in use is not put again with its module and fingerprint: %v", described.meta)
}
reads := described.reads
revision(4, shared)
k.judge()
if described.reads != reads {
t.Errorf("described-later: a second judging read %d entry objects again", described.reads-reads)
}
}
}
}
@@ -1,88 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running.<node>.<module>` is
// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every
// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it.
// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one.
func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module()}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) {
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
if err := w.hearRunning(m.Subject(), m.Data()); err != nil {
logf("%v", err)
return
}
wake()
}, logf)
if err != nil {
nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err)
return
}
w.mu.Lock()
w.runningHeard = true
w.mu.Unlock()
nh.Heard(nodeengine.StreamRunning)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
// hearRunning keeps one `running` event said on subject.
func (w *world) hearRunning(subject string, data []byte) error {
var r nodeengine.Running
if err := json.Unmarshal(data, &r); err != nil {
return fmt.Errorf("%s holds what is not a running: %w", subject, err)
}
if r.Node == "" || r.Module == "" {
return fmt.Errorf("%s names no node or no module, so it is kept for none", subject)
}
w.mu.Lock()
w.running[r.Node+"."+r.Module] = r
w.mu.Unlock()
return nil
}
// runningOf is the last `running` heard of module on node.
func (w *world) runningOf(node, module string) (nodeengine.Running, bool) {
w.mu.Lock()
defer w.mu.Unlock()
r, ok := w.running[node+"."+module]
return r, ok
}
// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the
// have).
func (w *world) runningNow() map[string]map[string]nodeengine.Running {
w.mu.Lock()
defer w.mu.Unlock()
if !w.runningHeard {
return nil
}
out := map[string]map[string]nodeengine.Running{}
for _, r := range w.running {
if out[r.Node] == nil {
out[r.Node] = map[string]nodeengine.Running{}
}
out[r.Node][r.Module] = r
}
return out
}
File diff suppressed because it is too large Load Diff
@@ -1,33 +1,23 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"os"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// The seat mesh-module-assigner's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
@@ -47,149 +37,129 @@ func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
}
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
// `<module>.<state>` where the manifest declares them under `reads`.
const haveState = "node-engine.have"
// The indexer's states the verbs read, each asked of the indexer's first-view tool of that name, never read from its bus
// state (novox/hq ADR 0596; ADR 0615 §1, ADR 0621 §1, §2: the manifest invokes them, and the indexer answers them to the
// assigner alone of the delivery's parts).
// `<module>.<state>`.
const (
indexerModules = "modules"
indexerManifests = "manifests"
modulesState = delivery.IndexerModule + ".modules"
haveState = "node-engine.have"
)
// indexerView is one of the indexer's states whole, asked of its first-view tool through ask (stdio.Ask, or a test's) page
// after page, each page continued after the one before's next. It is asked again at every call: the indexer's state is
// the truth, and the verbs keep no copy of it.
func indexerView(ask func(string, any) (json.RawMessage, error), state string) (map[string]json.RawMessage, error) {
tool := buckets.IndexerModule + "." + state
if ask == nil {
return nil, fmt.Errorf("%s cannot be asked here", tool)
}
page := views.AskVia(ask, tool)
out := map[string]json.RawMessage{}
started, pages := time.Now(), 0
for after := ""; ; {
a, err := page(context.Background(), after)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err)
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
}
pages++
maps.Copy(out, a.Values)
if a.Next == "" || a.Next == after {
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond))
return out, nil
}
after = a.Next
}
}
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
type seatTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
now func() time.Time
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatlist.Part
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
// direct listing.
watched *stdio.WatchedStates
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
return &seatTools{st: st, ask: ask, now: time.Now}
}
// keyed is one assignments key as a verb answers it.
// keyed is one placement key as a verb answers it.
type keyed struct {
Assignments mma.Assignments `json:"assignments"`
Revision uint64 `json:"revision"`
Placement delivery.Placement `json:"placement"`
Revision uint64 `json:"revision"`
}
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
all, err := readAll(s.st, mma.StateAssignments)
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
keys, err := s.st.keys(delivery.PlacementState)
if err != nil {
return nil, nil, fmt.Errorf("assignments cannot be read: %w", err)
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
}
keys := slices.Sorted(maps.Keys(all))
held := mmac.NewKeys()
sort.Strings(keys)
held := delivery.NewPlacementKeys()
var bad []string
for _, key := range keys {
if _, _, _, err := mmac.ParseKey(key); err != nil {
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
continue
}
if _, err := held.Put(key, all[key].raw, all[key].rev, false); err != nil {
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
if err != nil {
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
}
if !found {
continue
}
if _, err := held.Put(key, raw, rev, false); err != nil {
bad = append(bad, err.Error())
}
}
out, revs := held.AllRevisions()
if len(bad) > 0 && len(out) == 0 {
return nil, nil, fmt.Errorf("assignments cannot be read: %s", strings.Join(bad, "; "))
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
}
return out, revs, nil
}
// lays is what a proposal is checked against: the modules the indexer holds, as its first view of modules answers them,
// and the nodes with a record.
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
mods, err := indexerView(s.ask, indexerModules)
mods, err := s.st.keys(modulesState)
if err != nil {
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
}
held := map[string]bool{}
for k := range mods {
if name, err := meshcatalogclient.ParseModulesKey(k); err == nil {
for _, k := range mods {
if name, err := delivery.ParseModulesKey(k); err == nil {
held[name] = true
}
}
haves, err := readAll(s.st, haveState)
haves, err := s.st.keys(haveState)
if err != nil {
return nil, nil, fmt.Errorf("the nodes' records cannot be read: %w", err)
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
}
recorded := map[string]bool{}
for k, v := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if h, err := nodeengineclient.DecodeHaveNode(v.raw); err == nil && h.Record != nil {
recorded[node] = true
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
}
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
}
// proposal reads one open proposal.
func (s *seatTools) proposal(id string) (mma.Proposal, error) {
if err := mmac.CheckProposalID(id); err != nil {
return mma.Proposal{}, err
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
if err := delivery.CheckProposalID(id); err != nil {
return delivery.PlacementProposal{}, err
}
raw, _, found, err := s.st.get(mma.StateProposals, id)
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
if err != nil {
return mma.Proposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
}
if !found {
return mma.Proposal{}, fmt.Errorf("no proposal %s is open: mesh-module-assigner.proposals lists those that are", id)
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
}
return plc.DecodeProposal(raw)
return delivery.DecodePlacementProposal(raw)
}
// write writes a proposal laid by assign.Propose, whose write the assigner's process takes as the event to lay it: it writes the
// candidates, each with what it adds (design 47 G A-317, A-316); the verb writes none, so no node-engine hears a candidate before
// its parts.
func (s *seatTools) write(p mma.Proposal) error {
p.Written = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
// takes as the event to compose and judge it.
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
p.Written = s.now().UTC()
for _, node := range p.Nodes() {
key, err := delivery.PlacementCandidateKey(p.ID, node)
if err != nil {
return err
}
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
return fmt.Errorf("the candidate %s is not written: %w", key, err)
}
}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
}
return nil
@@ -197,17 +167,17 @@ func (s *seatTools) write(p mma.Proposal) error {
// proposed is what propose and adopt answer.
type proposed struct {
Proposal mma.Proposal `json:"proposal"`
Follow string `json:"follow"`
Proposal delivery.PlacementProposal `json:"proposal"`
Follow string `json:"follow"`
}
func followProposal(id string) string {
return fmt.Sprintf("the nodes it touches answer it; mesh-module-assigner.proposals with id %q says where it stands, and the "+
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
"operator is asked to approve it once its verdict passes", id)
}
// changesOf reads propose's changes: a JSON array, given as one or as its text.
func changesOf(v any) ([]mma.ProposalChange, error) {
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
var raw []byte
switch c := v.(type) {
case string:
@@ -220,7 +190,7 @@ func changesOf(v any) ([]mma.ProposalChange, error) {
return nil, err
}
}
var out []mma.ProposalChange
var out []delivery.PlacementProposalChange
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
@@ -236,92 +206,32 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
return nil, err
}
for _, c := range changes {
if c.Op == mmac.OpSeed {
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
if c.Op == delivery.OpSeed {
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
started := time.Now()
step := func(what string, err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s failed after %s: %v\n", by, what, time.Since(started).Round(time.Millisecond), err)
return
}
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s at %s\n", by, what, time.Since(started).Round(time.Millisecond))
}
current, revs, err := s.assignments()
step("assignments read", err)
current, revs, err := s.placements()
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
step("the indexer's modules and the nodes' records read", err)
if err != nil {
return nil, err
}
warrant, err := warrantOf(args["warrant"])
if err != nil {
return nil, err
}
enrollers := plc.EnrollersOf(current)
now := s.now()
p, _, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()})
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
findings = append(findings, refusal)
}
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
if len(findings) > 0 {
step(fmt.Sprintf("refused (%d findings)", len(findings)), nil)
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
step("judged", nil)
if err := s.write(p); err != nil {
step("the proposal "+p.ID+" written", err)
if err := s.write(p, laid); err != nil {
return nil, err
}
step("the proposal "+p.ID+" written", nil)
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
func warrantOf(v any) (*mma.Warrant, error) {
var raw []byte
switch w := v.(type) {
case nil:
return nil, nil
case string:
if strings.TrimSpace(w) == "" {
return nil, nil
}
raw = []byte(w)
default:
var err error
if raw, err = json.Marshal(w); err != nil {
return nil, err
}
}
var out mma.Warrant
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
}
return &out, nil
}
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: the holder of the seat
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
// `person.<module>`), else the caller as proven, which the refusal names.
func moduleOfCaller(by string, enrollers []string) string {
// The enrolment's caller stays (design 47 H-A-41): a module's bus user is `<node>.<module>`, a person's `person.<name>`.
_, module, ok := strings.Cut(by, ".")
if ok && !strings.HasPrefix(by, "person.") && slices.Contains(enrollers, module) {
return module
}
return by
}
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
// assignments-withdrawn carrying it whole.
// placement-withdrawn carrying it whole.
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
why := arg(args, "why")
if why == "" {
@@ -332,11 +242,11 @@ func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
return nil, err
}
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
raw, err := plc.EncodeProposal(p)
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
@@ -349,46 +259,107 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
if err != nil {
return nil, err
}
p.Again = ptr(s.now().UTC())
raw, err := plc.EncodeProposal(p)
p.Again = s.now().UTC()
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// doPlacement is the verb placement (design 47 G A-62, §704): the module's placements as the assignments rows hold them, one
// per branch a row names, with the nodes that run the module from it. A row naming no branch is one placement with the branch
// empty, as the row holds it.
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
module := arg(args, "module")
if module == "" {
return nil, errors.New("placement takes the module")
}
all, _, err := s.assignments()
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
current, _, err := s.placements()
if err != nil {
return nil, err
}
byBranch := map[string][]string{}
for node, a := range all {
for _, m := range a.Modules {
if m.Module == module {
byBranch[m.Branch] = append(byBranch[m.Branch], node)
if len(current) > 0 {
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
}
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
for _, id := range ids {
if p, err := s.proposal(id); err == nil && p.Seed {
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
}
}
}
out := mma.PlacementAnswer{Placements: []mma.ModulePlacement{}}
for _, branch := range slices.Sorted(maps.Keys(byBranch)) {
out.Placements = append(out.Placements, mma.ModulePlacement{Branch: branch, Nodes: slices.Sorted(slices.Values(byBranch[branch]))})
raw, err := s.ask(exportCall, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
}
return out, nil
export, err := assign.ExportOf(raw)
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// assignmentsAnswer is what the verb assignments answers.
type assignmentsAnswer struct {
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
// unreadable, is not handed away.
func (s *seatTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
// records the hand act and says placement-changed; every node returns to the controller's compositions.
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
if s.leased() {
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
}
why := arg(args, "why")
if why == "" {
return nil, errors.New("hand-back says why: it is recorded with the hand act")
}
if by == "" {
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
}
_, revs, err := s.placements()
if err != nil {
return nil, err
}
if len(revs) == 0 {
return nil, errors.New("there is no placement key to hand back")
}
now := s.now()
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the hand-back is not written: %w", err)
}
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
"node returns to the controller's compositions"}, nil
}
// placementAnswer is what the verb placement answers.
type placementAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Count int `json:"count"`
@@ -399,38 +370,38 @@ type assignmentsAnswer struct {
// historyEntry is one change of a key in the state's revisions.
type historyEntry struct {
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change plc.Change `json:"change"`
Deleted bool `json:"deleted,omitempty"`
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change delivery.PlacementChange `json:"change"`
Deleted bool `json:"deleted,omitempty"`
}
// doAssignments is the verb assignments (§10, §6): one node's key, one module's nodes, or every key; with history, the
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
// changes in order from the revisions the state keeps.
func (s *seatTools) doAssignments(args map[string]any) (any, error) {
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
if arg(args, "seats") == "true" {
return s.seats(arg(args, "node"))
}
all, revs, err := s.assignments()
all, revs, err := s.placements()
if err != nil {
return nil, err
}
node, module := arg(args, "node"), arg(args, "module")
out := assignmentsAnswer{Keys: map[string]keyed{}, Count: len(all), History: plc.AssignmentsHistory}
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
for n, p := range all {
if node != "" && n != node {
continue
}
if module != "" && !strings.Contains(" "+strings.Join(mmac.Assigned(p), " ")+" ", " "+module+" ") {
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
continue
}
out.Keys[n] = keyed{Assignments: p, Revision: revs[n]}
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
out.Nodes = append(out.Nodes, n)
}
sort.Strings(out.Nodes)
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
return nil, fmt.Errorf("%s has no assignments key", node)
return nil, fmt.Errorf("%s has no placement key", node)
}
if arg(args, "history") != "true" {
return out, nil
@@ -468,8 +439,8 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
out.Changes = append(out.Changes, e)
}
}
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its assignments-changed event on "+
"the seat's subject on the shared events stream, kept %d days", plc.AssignmentsHistory, plc.EventDays)
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
return out, nil
}
@@ -477,38 +448,42 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
Proposal delivery.PlacementProposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
func (s *seatTools) doProposals(args map[string]any) (any, error) {
ids, err := s.st.keys(mma.StateProposals)
ids, err := s.st.keys(delivery.PlacementProposalsState)
if err != nil {
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
}
cands, err := s.st.keys(mma.StateCandidates)
cands, err := s.st.keys(delivery.PlacementCandidatesState)
if err != nil {
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
}
haves, err := readAll(s.st, haveState)
haves, err := s.st.keys(haveState)
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be read: %w", err)
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
for k, v := range haves {
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
h, err := nodeengineclient.DecodeHaveNode(v.raw)
raw, _, found, err := s.st.get(haveState, k)
if err != nil || !found {
continue
}
h, err := delivery.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]nodeengine.CandidateAnswer{}
answers[id] = map[string]delivery.CandidateAnswer{}
}
answers[id][node] = a
}
@@ -525,11 +500,11 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
}
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
for _, k := range cands {
pid, node, err := plc.ParseCandidateKey(k)
pid, node, err := delivery.ParsePlacementCandidateKey(k)
if err != nil || pid != id {
continue
}
if _, rev, found, err := s.st.get(mma.StateCandidates, k); err == nil && found {
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
o.Candidates[node] = rev
}
}
@@ -544,61 +519,50 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
// only from a tool of that name), with the seat's own schemas.
func (s *seatTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
input := map[string]map[string]any{}
description := map[string]string{}
for _, v := range seat.Serves {
input[v.Name], description[v.Name] = v.Input, v.Description
}
run := map[string]func(map[string]any) (any, error){
mma.VerbAssignments: s.doAssignments,
mma.VerbProposals: s.doProposals,
mma.VerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
mma.VerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
mma.VerbAgain: s.doAgain,
mma.VerbPlacement: s.doPlacement,
catalogue.PlacementVerbPlacement: s.doPlacement,
catalogue.PlacementVerbProposals: s.doProposals,
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbAgain: s.doAgain,
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
}
var out []stdio.Tool
for _, v := range mma.Verbs() {
out = append(out, stdio.Tool{Name: mma.Seat + "." + v, Run: run[v]})
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
}
return out
}
// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
func (s *seatTools) seatHolders() (map[string][]byte, error) {
if s.watched != nil {
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
return vals, nil
}
}
keys, err := s.st.keys(mma.StateSeatHolders)
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
keys, err := s.st.keys(delivery.SeatHoldersState)
if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
}
out := make(map[string][]byte, len(keys))
sort.Strings(keys)
out := []delivery.SeatHolders{}
for _, key := range keys {
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
}
if found {
out[key] = raw
}
}
return out, nil
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
raws, err := s.seatHolders()
if err != nil {
return nil, err
}
keys := slices.Sorted(maps.Keys(raws))
out := []mma.SeatHolders{}
for _, key := range keys {
var sh mma.SeatHolders
if json.Unmarshal(raws[key], &sh) != nil {
if !found {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
var sh delivery.SeatHolders
if json.Unmarshal(raw, &sh) != nil {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
continue
}
out = append(out, sh)
@@ -606,56 +570,76 @@ func (s *seatTools) seats(node string) (any, error) {
return map[string]any{"seats": out, "holds": len(keys)}, nil
}
// keyedRevision is one revision of a node's assignments key, the key it is of beside it.
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
type keyedRevision struct {
stdio.StateRevision
Key string
}
// nodeHistory is the revisions of a node's assignments, in the order the bus wrote them: its node key's `<node>.node`,
// whose every revision is one write of the node (research 072 row 37: the node key is a write's commit, so its history is
// the node's assignments history).
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
nodeKey, err := mmac.NodeKey(node)
legacy, err := delivery.LegacyPlacementKey(node)
if err != nil {
return nil, err
}
revisions, err := s.st.history(mma.StateAssignments, nodeKey)
nodeKey, err := delivery.PlacementNodeKey(node)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", nodeKey, err)
return nil, err
}
var out []keyedRevision
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: nodeKey})
for _, key := range []string{legacy, nodeKey} {
revisions, err := s.st.history(delivery.PlacementState, key)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
}
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: key})
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
return out, nil
}
// sketch is a revision of a node's assignments as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (plc.ChangeOf).
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (delivery.ChangeOf).
type sketch struct {
modules []mma.AssignmentsByNodeNodeValueModulesItem
modules []delivery.PlacementRef
nodeWide string
}
// sketchOf reads one revision of a node key; a value that does not read places nothing.
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
func sketchOf(key string, value []byte) sketch {
_, _, kind, err := mmac.ParseKey(key)
if err != nil || kind != mmac.NodeKind {
return sketch{}
}
n, err := mmac.DecodeNode(value)
_, _, kind, err := delivery.ParsePlacementKey(key)
if err != nil {
return sketch{}
}
raw, _ := json.Marshal([]any{n.Pins, n.Holds})
return sketch{modules: n.Modules, nodeWide: string(raw)}
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
raw, _ := json.Marshal([]any{pins, holds})
return string(raw)
}
if kind == delivery.PlacementNodeKind {
n, err := delivery.DecodePlacementNode(value)
if err != nil {
return sketch{}
}
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
}
p, err := delivery.DecodePlacement(value)
if err != nil {
return sketch{}
}
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
for _, m := range p.Modules {
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
}
return out
}
// changeFrom is what changed from before to s on node, as plc.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) plc.Change {
c := plc.Change{Node: node}
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
c := delivery.PlacementChange{Node: node}
was := map[string]string{}
for _, r := range before.modules {
was[r.Module] = r.Digest
File diff suppressed because it is too large Load Diff
@@ -1,7 +0,0 @@
package main
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
// (plc.CheckAssignments), and seat-holders is not written.
@@ -1,114 +0,0 @@
package main
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
)
// testDefinitions are seatlisttest's, parsed once.
var testDefinitions = sync.OnceValue(func() []seatdef.Definition { return seatlisttest.Definitions() })
// standInCatalog is the seat mesh-catalog as a test's reader asks it (ADR 0636 §4): every definition of seatlisttest, or,
// silent, no answer at all.
type standInCatalog struct{ silent bool }
func (c standInCatalog) Ask(key string, body any) (json.RawMessage, error) {
if c.silent || key != seatlist.Key() {
return nil, errors.New("no responders")
}
defs := testDefinitions()
seat, _ := body.(map[string]any)["seat"].(string)
if seat == "" {
l := seatlist.Listing{Revision: 1}
for _, d := range defs {
l.Seats = append(l.Seats, seatlist.Entry{Seat: d.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + d.Seat})
}
return json.Marshal(l)
}
for _, d := range defs {
if d.Seat == seat {
raw, err := json.Marshal(d)
if err != nil {
return nil, err
}
return json.Marshal(seatlist.Answer{Seat: seat, Found: true, Revision: 1,
Entry: &seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "d-" + seat, Definition: raw}})
}
}
return json.Marshal(seatlist.Answer{Seat: seat, Revision: 1})
}
func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
// gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list
// (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere.
func gateOver(t *testing.T, silent bool) *seatlist.Part {
t.Helper()
if silent {
// A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test
// that waits for it to stay unread keeps short.
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
}
g := seatlist.NewPart("assigner")
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
if r == nil {
t.Fatal("the reader did not start")
}
<-r.Ready()
if g.Heard() == silent {
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health())
}
return g
}
// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the
// assigner's code finds no start of a reader and no tools of its own.
func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} {
if strings.Contains(string(raw), own) {
t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own)
}
}
}
}
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
if _, ok := seatlist.Definition(seatlist.NewPart("assigner").Reader, "mesh-wants")(); ok {
t.Error("a definition is answered before the seat list's reader started")
}
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
t.Error("a definition is answered while the seat mesh-catalog has not answered the list")
}
read := gateOver(t, false)
if d, ok := seatlist.Definition(read.Reader, "mesh-wants")(); !ok || d.Seat != "mesh-wants" {
t.Errorf("the read list's mesh-wants is answered as %v, %v", d.Seat, ok)
}
if _, ok := seatlist.Definition(read.Reader, "no-such-seat")(); ok {
t.Error("a seat the list does not hold is answered a definition")
}
}
@@ -1,18 +0,0 @@
package main
import (
"strings"
"testing"
)
// A module a candidate adds with no release heard is said once, not each pass.
func TestASkippedAddIsSaidOncePerModuleAndFingerprint(t *testing.T) {
var logs []string
p := &passer{logf: func(f string, a ...any) { logs = append(logs, strings.TrimSpace(f)) }}
p.skippedAdd("n", "m", "fp", "why")
p.skippedAdd("n", "m", "fp", "why")
p.skippedAdd("n", "m", "fp2", "why")
if len(logs) != 2 {
t.Fatalf("said %d times, want 2: %v", len(logs), logs)
}
}
@@ -1,318 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2,
// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's
// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key
// <seat>.<ask>, for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an
// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask
// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped.<ask> for
// ask-taken-bound, and either ends its step could not run naming the drop and who asked it.
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
const (
askTakenBoundSetting = "ask-taken-bound"
defaultAskTakenBound = 10 * time.Minute
)
// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404).
const droppedPrefix = "dropped."
// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet.
type takenStore interface {
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
Get(key string) (*stdio.StateEntry, error)
RenewTTL(key string, value any, ttl time.Duration) (uint64, error)
Delete(key string) error
}
// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg).
type queueMsg interface {
Data() []byte
Headers() nats.Header
Ack() error
Nak() error
Term() error
InProgress() error
}
// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end.
type askDrop struct {
By string `json:"by,omitempty"`
Why string `json:"why"`
At time.Time `json:"at"`
}
func (d *askDrop) Error() string {
by := d.By
if by == "" {
by = "its asker"
}
return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why)
}
// askIDOf is the ask's id a put carries: its event id, else its message id.
func askIDOf(h nats.Header) string {
if id := h.Get("x-event-id"); id != "" {
return id
}
return h.Get(nats.MsgIdHdr)
}
// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is
// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed
// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken
// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run
// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice.
func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) {
id := askIDOf(m.Headers())
var run meshwants.StepRun
err := json.Unmarshal(m.Data(), &run)
how := c.howOf(run.Step, false)
if err != nil || id == "" || how == nil {
c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err)
_ = m.Term()
return
}
if d := c.droppedBefore(id); d != nil {
c.hold(id, m)
c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d)
c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "")
if err := c.taken.Delete(droppedPrefix + id); err != nil {
c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err)
}
return
}
if c.running(id) {
c.hold(id, m)
c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
return
}
first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound)
if err != nil {
c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
_ = m.Nak()
return
}
c.hold(id, m)
if first {
at := c.now().UTC()
if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil {
c.logf("saying ask-taken of %s: %v", id, err)
}
} else {
c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline)
}
go c.say1(ctx, id, run, how, "")
}
// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs.
func (c *compositionSteps) hold(id string, m queueMsg) {
c.mu.Lock()
defer c.mu.Unlock()
if c.held == nil {
c.held = map[string][]queueMsg{}
}
c.held[id] = append(c.held[id], m)
}
// settle acknowledges (ack) or hands again (nak) every message held for id.
func (c *compositionSteps) settle(id string, ack bool) {
c.mu.Lock()
ms := c.held[id]
delete(c.held, id)
c.mu.Unlock()
for _, m := range ms {
if ack {
_ = m.Ack()
} else {
_ = m.Nak()
}
}
}
// inProgressEvery says every message held for id in progress every d, until the returned stop is called.
func (c *compositionSteps) inProgressEvery(id string) (stop func()) {
every := time.Duration(0)
if c.inProgress != nil {
every = c.inProgress()
}
if every <= 0 {
return func() {}
}
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(every)
done, ended := make(chan struct{}), make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
c.mu.Lock()
ms := append([]queueMsg(nil), c.held[id]...)
c.mu.Unlock()
for _, m := range ms {
_ = m.InProgress()
}
}
}
}()
return func() { close(done); <-ended }
}
// droppedBefore is the drop kept for an ask not pulled yet, nil for none.
func (c *compositionSteps) droppedBefore(id string) *askDrop {
if c.taken == nil {
return nil
}
e, err := c.taken.Get(droppedPrefix + id)
if err != nil {
c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err)
return nil
}
if e == nil {
return nil
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
return nil
}
return &d
}
// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and
// ends could not run naming it; one not running here is kept in taken as dropped.<ask> for ask-taken-bound, so its pull ends
// it the same way. It answers whether the ask was held here.
func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) {
id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why)
if id == "" || why == "" {
return nil, errors.New("give the ask's id and why it is dropped")
}
by := strings.TrimSpace(in.By)
if by == "" {
by = caller
}
d := &askDrop{By: by, Why: why, At: c.now().UTC()}
c.mu.Lock()
stop := c.stops[id]
c.mu.Unlock()
if stop != nil {
stop(d)
c.logf("the ask %s running here is dropped: %v", id, d)
return meshwants.DropAnswer{"held": true}, nil
}
if c.taken == nil {
return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id)
}
if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil {
return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err)
}
c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d)
return meshwants.DropAnswer{"held": false}, nil
}
// dropVerb is the process's handler of the seat mesh-wants' act drop.
func dropVerb(c *compositionSteps) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.DropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.dropAsk(caller, in)
},
}
}
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401).
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
return func() time.Duration {
if d, ok := def(); ok {
if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
return time.Duration(worker.InProgressSeconds) * time.Second
}
}
return ackWait() / 3
}
}
// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error {
stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)
var once sync.Once
bind := func() bool {
cons, err := js.Consumer(ctx, stream, worker)
if err != nil {
return false
}
bound := false
once.Do(func() {
bound = true
info := cons.CachedInfo()
if info != nil && info.Config.AckWait > 0 {
ackWait(info.Config.AckWait)
}
pending := 1000
if info != nil && info.Config.MaxAckPending > 0 {
pending = info.Config.MaxAckPending
}
cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending))
if err != nil {
c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err)
return
}
c.logf("the composition step is pulled from %s on %s", worker, stream)
go func() { <-ctx.Done(); cc.Stop() }()
})
return bound
}
if bind() {
return nil
}
c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream)
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
if err != nil {
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err)
}
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
if bind() { // made between the first look and the subscription
_ = sub.Unsubscribe()
}
return nil
}
// askTakenBound reads the setting ask-taken-bound (§781).
func askTakenBound() (time.Duration, error) {
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
@@ -1,69 +0,0 @@
package main
import (
"errors"
"fmt"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// kept is one key's value and revision, as a whole state read gives it.
type kept struct {
raw []byte
rev uint64
}
// pager is a store that reads a state's values many to a request (stdio.KeptState.Page): the runtime's store.
type pager interface {
page(state, after string) (stdio.StatePage, error)
}
// pageBytes and pageBudget bound one page of a whole state read, so its answer fits one bus message and its time.
const (
pageBytes = 768 * 1024
pageBudget = 10 * time.Second
)
func (runtimeStore) page(state, after string) (stdio.StatePage, error) {
return stdio.State(state).Page(after, pageBytes, pageBudget)
}
// readAll is every key of a state with its value and revision: in pages when the store has them, else key by key. Key by
// key cost one bus request each, so the assignments (about 330 keys) took 27 s to read and a proposal outlasted its
// caller's 30 s (2026-11-14), as the indexer's view of the trunk had the hour before.
func readAll(st store, state string) (map[string]kept, error) {
out := map[string]kept{}
if p, ok := st.(pager); ok {
for after := ""; ; {
pg, err := p.page(state, after)
if errors.Is(err, errors.ErrUnsupported) {
break
}
if err != nil {
return nil, fmt.Errorf("%s cannot be read: %w", state, err)
}
for _, e := range pg.Entries {
out[e.Key] = kept{raw: e.Value, rev: e.Revision}
}
if pg.Next == "" || pg.Next == after {
return out, nil
}
after = pg.Next
}
}
keys, err := st.keys(state)
if err != nil {
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
}
for _, k := range keys {
raw, rev, found, err := st.get(state, k)
if err != nil {
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
}
if found {
out[k] = kept{raw: raw, rev: rev}
}
}
return out, nil
}
@@ -1,360 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
)
// One pipeline's wants are written tier by tier (design 47 H §751, H-A-10, H-A-56): on a passed pipeline-ended the assigner
// writes the wants of its first tier's movers; once each of them runs at the mover's fingerprint (the node-engine seat's
// `running`, H-A-40) on every present placing machine, it writes the next tier's, and so on. The tier progress is the key
// `pipeline.<branch>.<source>` of the state written, `{tier, since, movers}`; a tier counts placed movers only, and a machine away
// is not waited for. A tier whose movers do not all run within `tier-bound` ends the delivery loudly: no later tier is written,
// the finding `tier-late` (scope delivery, id the pipeline) is raised naming the movers and machines not running, and
// `pipeline-wanted` says outcome tier-late; once every tier runs it says outcome running and the key goes (H-A-75). tier-late
// clears when the late movers run or a later passed pipeline moves them (H-A-56).
// tierPrefix starts the key of a pipeline's tier progress in the state written.
const tierPrefix = "pipeline."
// kindTierLate is the finding of a tier not running within tier-bound (design 47 H §751).
const kindTierLate = "tier-late"
// defaultTierBound is tier-bound's default, 30 minutes (design 47 H §751).
const defaultTierBound = 30 * time.Minute
// tierMover is one mover of a passed pipeline whose version the state written keeps: its module, fingerprint and tier.
type tierMover struct {
Module string `json:"module"`
Fp string `json:"fp"`
Tier int64 `json:"tier"`
// NoVersion is a passed mover whose version is not yet written: its tier does not run until it is (H-A-277 (1)).
NoVersion bool `json:"no-version,omitempty"`
}
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source
// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a
// value kept before the field, read as the key's number.
type tierProgress struct {
Tier int64 `json:"tier"`
Since time.Time `json:"since"`
Movers []tierMover `json:"movers"`
Source int64 `json:"source,omitempty"`
}
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
func tierKey(pipeline string) string { return tierPrefix + pipeline }
// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's
// end has two).
func isTierKey(key string) bool {
return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2
}
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
// lowest of theirs, since now (H-A-277 (1)). A pipeline heard again adds to the progress that stands each mover it does not hold,
// at the mover's tier, and marks versioned each held mover whose version is now written (H-A-270, H-A-273): its want is written
// when the progress reaches its tier, or at once when that tier ran already, never out of tier order.
func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover, versioned map[string]bool) error {
for {
t, rev, ok := e.tiersOf(pipeline)
if !ok {
if len(movers) == 0 {
return nil
}
first := movers[0].Tier
for _, m := range movers {
first = min(first, m.Tier)
}
t = tierProgress{Tier: first, Since: e.clock().UTC(), Source: source}
}
moved := false
for i := range t.Movers {
if t.Movers[i].NoVersion && versioned[t.Movers[i].Module] {
t.Movers[i].NoVersion, moved = false, true
}
}
for _, m := range movers {
if !slices.ContainsFunc(t.Movers, func(h tierMover) bool { return h.Module == m.Module }) {
t.Movers, moved = append(t.Movers, m), true
}
}
if !moved {
return nil
}
value, err := json.Marshal(t)
if err != nil {
return err
}
err = e.st.swap(tierKey(pipeline), value, rev)
if errors.Is(err, errSwapped) {
if _, _, held := e.tiersOf(pipeline); !ok && !held {
return nil // a key that stands but does not read is left as it is
}
continue
}
if err == nil && ok {
e.logf("pipeline %s: a mover whose version came after its end is taken into the tier progress", pipeline)
}
return err
}
}
func (e *pipelineEnds) clock() time.Time {
if e.now != nil {
return e.now()
}
return time.Now()
}
// tiersOf is the tier progress of pipeline, with its revision; false when none stands.
func (e *pipelineEnds) tiersOf(pipeline string) (tierProgress, uint64, bool) {
raw, rev, err := e.st.get(tierKey(pipeline))
var t tierProgress
if err != nil || raw == nil || json.Unmarshal(raw, &t) != nil {
return tierProgress{}, 0, false
}
return t, rev, true
}
// tierWritten says whether the want of an end may be written: its pipeline's tier progress reached the end's rank, or no
// progress stands for its pipeline (every tier ran).
func (e *pipelineEnds) tierWritten(end End) bool {
t, _, ok := e.tiersOf(end.Pipeline)
return !ok || end.Rank <= t.Tier
}
// tierBound is the setting tier-bound; zero or less is its default.
func (p *passer) tierBoundOf() time.Duration {
if p.tierBound > 0 {
return p.tierBound
}
return defaultTierBound
}
// tiers moves each pipeline's tier progress (design 47 H §751): a tier whose placed movers all run on every present placing machine
// gives way to the next, the last to the end said `running`; a tier past tier-bound is late. placedOn is, by module, the nodes
// placing it on branch. It returns the earliest deadline still to come (zero: none), for one timer to wake the pass at.
func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []string) time.Time {
if p.ends == nil {
return time.Time{}
}
ks, err := p.ends.st.keys()
if err != nil {
p.logf("the tier progress cannot be listed: %v", err)
return time.Time{}
}
sort.Strings(ks)
now := p.clock().UTC()
bound := p.tierBoundOf()
var next time.Time
for _, key := range ks {
if !isTierKey(key) {
continue
}
pipeline := strings.TrimPrefix(key, tierPrefix)
branchEsc, number, _ := strings.Cut(pipeline, ".")
branch, err := keys.UnescapeName(branchEsc)
if err != nil {
continue
}
for {
t, rev, ok := p.ends.tiersOf(pipeline)
if !ok {
break
}
notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier)
late := !now.Before(t.Since.Add(bound))
if late {
p.tierLate(pipeline, branch, number, t, notRunning)
break
}
if len(notRunning) > 0 {
if d := t.Since.Add(bound); next.IsZero() || d.Before(next) {
next = d
}
break
}
// The tier runs: the next tier's wants are written, or every tier ran.
later := int64(0)
for _, m := range t.Movers {
if m.Tier > t.Tier && (later == 0 || m.Tier < later) {
later = m.Tier
}
}
if later == 0 {
if err := p.ends.st.drop(key); err != nil {
p.logf("the tier progress %s is not deleted: %v", key, err)
break
}
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil)
break
}
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source})
if err == nil {
err = p.ends.st.swap(key, value, rev)
}
if errors.Is(err, errSwapped) {
continue
}
if err != nil {
p.logf("the tier progress %s is not moved to tier %d: %v", key, later, err)
break
}
p.logf("pipeline %s: tier %d runs on every present placing machine, so tier %d's wants are written", pipeline, t.Tier, later)
}
}
return next
}
// notRunning is, of tier's movers in t whose kept end still names pipeline, those not running at their fingerprint on a present
// placing machine, with those machines. A mover a later pipeline moved counts as run; one placed nowhere counts as run.
func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) []string, pipeline, branch string, t tierProgress, tier int64) []meshwants.NotRunning {
var out []meshwants.NotRunning
for _, m := range t.Movers {
if m.Tier != tier {
continue
}
if m.NoVersion {
// A mover whose version is not yet written holds its tier, under the tier bound (H-A-277 (1)).
machines := []string{}
for _, node := range placedOn(m.Module, branch) {
if online, known := n.Online[node]; !known || online {
machines = append(machines, node)
}
}
sort.Strings(machines)
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
continue
}
if end, ok := p.ends.endOf(m.Module, branch); !ok || end.Pipeline != pipeline {
continue // a later pipeline moved it
}
var machines []string
for _, node := range placedOn(m.Module, branch) {
if online, known := n.Online[node]; known && !online {
continue // a machine away is not waited for
}
r, ok := n.Running[node][m.Module]
if !ok || r.Stopped || r.Fingerprint != m.Fp {
machines = append(machines, node)
}
}
if len(machines) > 0 {
sort.Strings(machines)
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
}
}
return out
}
// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late,
// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is
// written, until no later tier's mover is kept at this pipeline any more.
func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) {
o := tierLateFinding(pipeline, t.Tier, notRunning)
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
if len(notRunning) > 0 {
if !p.said[key+".raised"] {
if p.findings.Put != nil {
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", key, err)
return
}
}
p.said[key+".raised"] = true
delete(p.said, key+".cleared")
p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary)
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
}
} else if !p.said[key+".cleared"] {
if p.findings.Delete != nil {
if err := p.findings.Delete(key); err != nil {
p.logf("the finding %s is not deleted: %v", key, err)
return
}
}
p.said[key+".cleared"] = true
delete(p.said, key+".raised")
}
for _, m := range t.Movers {
if m.Tier <= t.Tier {
continue
}
if end, ok := p.ends.endOf(m.Module, branch); ok && end.Pipeline == pipeline {
return // a later tier's mover still kept at this pipeline: its want stays unwritten
}
}
if len(notRunning) == 0 {
if err := p.ends.st.drop(tierKey(pipeline)); err != nil {
p.logf("the tier progress of %s is not deleted: %v", pipeline, err)
}
}
}
// tierLateFinding is the finding tier-late of pipeline's tier, naming the movers and machines not running (design 47 H §751).
func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunning) assign.Observed {
var named []string
for _, nr := range notRunning {
named = append(named, fmt.Sprintf("%s on %s", nr.Module, strings.Join(nr.Machines, ", ")))
}
summary := fmt.Sprintf("pipeline %s: tier %d does not run within tier-bound: %s", pipeline, tier, strings.Join(named, "; "))
return assign.Observed{Scope: mcc.ScopeDelivery, ID: pipeline, Kind: kindTierLate, Severity: mcseat.FindingSeverityWarning,
Summary: summary, Headline: "A delivery's tier does not run in time",
Explanation: summary + ". The assigner writes no later tier of this pipeline; the delivery ends here.",
Source: "assigner." + kindTierLate}
}
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its
// revision the number of the pipeline's key `<branch>.<revision>`, its source the one the pipeline-ended named (design 47
// H-A-214, H-A-216), the key's number for a progress kept before it held one.
func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) {
var revision int64
_, _ = fmt.Sscan(number, &revision)
if source <= 0 {
source = revision
}
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning}
event := meshwants.EventPipelineWanted + "." + pipeline
if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil {
p.logf("pipeline-wanted of %s is not said: %v", pipeline, err)
}
}
// tierDeadline sets the one timer that wakes the pass at the earliest tier's deadline (H-A-56); zero stops it.
func (p *passer) tierDeadline(at time.Time) {
if p.tierWake == nil {
return
}
if p.tierTimer != nil {
p.tierTimer.Stop()
}
if at.IsZero() {
return
}
p.tierTimer = time.AfterFunc(max(at.Sub(p.clock()), 0), p.tierWake)
}
// tierBoundSetting is the setting tier-bound in minutes (design 47 H §751), read at the start from the file the manifest declares
// with the content ${config:tier-bound}; zero, its default, for a file missing or empty.
func tierBoundSetting(getenv func(string) string) (time.Duration, error) {
minutes, err := settingFile(getenv("MESH_ASSIGNER_TIER_BOUND_FILE"))
return time.Duration(minutes) * tierBoundUnit, err
}
// tierBoundUnit is the unit of the setting tier-bound, a minute; the whole test sets a shorter one, so a tier passes its bound
// without a test waiting minutes.
var tierBoundUnit = time.Minute
@@ -1,210 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
)
// A7/AC1, A7/AC2 (design 47 H §751, H-A-56): a pipeline of a provider (tier 1) and its consumer (tier 2) writes the provider's
// wants first and the consumer's only once the provider runs at its fingerprint on every present placing machine (one machine
// away is not waited for), then says pipeline-wanted running and drops its progress; a provider that never runs is late at
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
for _, runs := range []bool{true, false} {
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
now := start
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
p.now = func() time.Time { return now }
p.say = func(oc.Condition) error { return nil }
findings := map[string]mcseat.Finding{}
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { findings[views.FindingKey(f.Scope, f.ID, f.Kind)] = f; return nil },
Delete: func(key string) error { delete(findings, key); return nil }}
var said []meshwants.PipelineWanted
p.publishWants = func(event string, body any, _ string) error {
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
if event != "pipeline-wanted."+trunkBranch+".5" {
t.Errorf("pipeline-wanted said as %s", event)
}
said = append(said, b)
}
return nil
}
for i, m := range []struct{ name, fp string }{{"db", db}, {"web", web}} {
digest := strings.Repeat(string(rune('e'+i)), 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.fp, Manifest: digest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, m.name, mod)
p.w.manifests[digest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
}
placeOn(t, p.w, "g14", "db", "web")
placeOn(t, p.w, "ace", "db", "web")
p.w.heardPresence("ace", true, start) // away: not waited for
p.w.mu.Lock()
p.w.runningHeard = true
p.w.mu.Unlock()
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
e.Source = 4
if _, err := p.ends.take(e); err != nil {
t.Fatal(err)
}
wanted := func(node, module string) bool {
key, _ := mwc.WantKey(node, module)
_, ok := wantsKV.values[key]
return ok
}
p.wants(context.Background())
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values)
}
if runs {
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
if err := p.w.hearRunning("running.g14.db", raw); err != nil {
t.Fatal(err)
}
now = start.Add(time.Minute)
p.wants(context.Background())
if !wanted("g14", "web") || !wanted("ace", "web") {
t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values)
}
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
_ = p.w.hearRunning("running.g14.web", raw)
p.wants(context.Background())
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
}
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
t.Error("the tier progress stands after every tier ran")
}
continue
}
now = start.Add(29 * time.Minute)
p.wants(context.Background())
if len(findings) != 0 || len(said) != 0 {
t.Fatalf("late before tier-bound: %v, %+v", findings, said)
}
now = start.Add(30 * time.Minute)
p.wants(context.Background())
p.wants(context.Background())
f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"]
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
}
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" ||
said[0].Revision != 5 || said[0].Source != 4 {
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
}
// A tier progress kept before it held its source says the key's number as both (H-A-216).
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
}
if wanted("g14", "web") || wanted("ace", "web") {
t.Error("a consumer's want is written after its provider's tier was late")
}
}
}
// H-A-270, H-A-273, H-A-277 (1): a pipeline's tier progress holds every passed mover, versioned or not, and a tier with a mover
// whose version is not yet written does not run until it is: no later tier is wanted out of order. When the lowest tier's version
// comes last, nothing is wanted before it; with db (1) and api (3) versioned and web (2) late, api is wanted only once web runs.
func TestALateMoverHoldsItsTier(t *testing.T) {
type step struct {
versions map[string]string // a pipeline-ended taken, each mover's version ("": none yet)
runs string // or: this module runs on g14
wanted string // then: the modules wanted, in order of the list
}
for _, c := range []struct {
name string
steps []step
}{
{"the lowest tier's version comes last", []step{
{versions: map[string]string{"db": "", "web": "0.1.2", "api": "0.3.0"}, wanted: ""},
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db"},
{runs: "db", wanted: "db,web"},
{runs: "web", wanted: "db,web,api"},
}},
{"a middle tier's version comes late", []step{
{versions: map[string]string{"db": "0.2.0", "web": "", "api": "0.3.0"}, wanted: "db"},
{runs: "db", wanted: "db"},
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db,web"},
{runs: "web", wanted: "db,web,api"},
}},
} {
t.Run(c.name, func(t *testing.T) {
fps := map[string]string{}
tiers := map[string]int64{"db": 1, "web": 2, "api": 3}
for i, m := range []string{"db", "web", "api"} {
fps[m] = meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i)), 64)
}
start := time.Date(2026, 11, 21, 4, 0, 0, 0, time.UTC)
now := start
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
p.now = func() time.Time { return now }
p.say = func(oc.Condition) error { return nil }
p.publishWants = func(string, any, string) error { return nil }
for i, m := range []string{"db", "web", "api"} {
digest := strings.Repeat(string(rune('1'+i)), 64)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fps[m], Manifest: digest, Repository: "novox/" + m, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, m, mod)
p.w.manifests[digest] = parsed{module: m, manifest: catalogue.Manifest{Module: m}}
}
placeOn(t, p.w, "g14", "db", "web", "api")
p.w.mu.Lock()
p.w.runningHeard = true
p.w.mu.Unlock()
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
for i, s := range c.steps {
if s.versions != nil {
var movers []mpr.MoverEnded
for _, m := range []string{"db", "web", "api"} {
movers = append(movers, mover(m, fps[m], s.versions[m], tiers[m], mpr.StepResultOutcomePassed))
}
if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, movers...)); err != nil {
t.Fatal(err)
}
} else {
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: s.runs, Fingerprint: fps[s.runs]})
if err := p.w.hearRunning("running.g14."+s.runs, raw); err != nil {
t.Fatal(err)
}
}
now = now.Add(time.Minute)
p.wants(context.Background())
var got []string
for _, m := range []string{"db", "web", "api"} {
key, _ := mwc.WantKey("g14", m)
if _, ok := wantsKV.values[key]; ok {
got = append(got, m)
}
}
if strings.Join(got, ",") != s.wanted {
t.Fatalf("step %d: wanted %v, want %q", i+1, got, s.wanted)
}
}
})
}
}
@@ -1,14 +1,9 @@
package main
import (
plc "mesh-delivery-assigner/internal/assignment"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/busstate"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/puts"
"os"
"regexp"
"slices"
@@ -17,20 +12,13 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
// (seat_tools.go, ADR 0567 §10).
// putsState is the assigner's state of puts, as the runtime names a module's own state.
@@ -48,9 +36,7 @@ type store interface {
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
type putTools struct {
st store
// ask asks a module's tool (stdio.Ask, or a test's): the indexer's first view of modules for newest.
ask func(key string, body any) (json.RawMessage, error)
st store
now func() time.Time
mu sync.Mutex
// heard is every put the watch handed, by key; changed is closed and replaced at each.
@@ -58,8 +44,8 @@ type putTools struct {
changed chan struct{}
}
func newPutTools(st store, ask func(string, any) (json.RawMessage, error)) *putTools {
return &putTools{st: st, ask: ask, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
func newPutTools(st store) *putTools {
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
}
// hear takes a put's change in from the watch.
@@ -71,66 +57,72 @@ func (s *putTools) hear(key string, value []byte) {
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole.
var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181:
// a put names the whole fingerprint, the builder's builds being no bucket to list).
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
case v == "newest":
// Asked of the indexer's first view of modules, never read from its bus state (ADR 0596, ADR 0621; issue 1966).
mods, err := indexerView(s.ask, indexerModules)
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
if err != nil {
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
raw, found := mods[module]
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := meshcatalogclient.DecodeModule(raw)
m, err := delivery.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.Fp2 == "" {
if m.FP2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.Fp2, nil
return m.FP2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version)
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
puts.Put
delivery.Put
Follow string `json:"follow,omitempty"`
// Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the
// node's want until it ends or is released.
Older string `json:"older-than-newest,omitempty"`
// Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is
// written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*).
Since time.Time `json:"since"`
Bound string `json:"bound,omitempty"`
}
// answerOf is what assigner_put and assigner_puts answer of the put p of module on node.
func answerOf(node, module string, p puts.Put) putAnswer {
a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()}
if putWaits(p) {
a.Bound = putWaitBound.String()
}
return a
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -154,17 +146,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before.FP == fp && before.Hold == hold {
a := answerOf(node, module, before)
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := puts.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, puts.Asked, words)
data, err := puts.Encode(p)
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -172,21 +162,19 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
a := answerOf(node, module, s.await(key, p))
a.Follow = followPut(node, module)
return a, nil
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p puts.Put) puts.Put {
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := puts.Decode(raw); err == nil && now.At.Equal(p.At) {
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != puts.Asked && now.Step != puts.Build && now.Step != puts.Compose {
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
return p // it waits on a person or the node, or ended
}
}
@@ -206,11 +194,8 @@ func followPut(node, module string) string {
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if arg(args, "how") != "" {
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
}
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
key, err := delivery.PutKey(node, module)
if err != nil {
return nil, err
}
@@ -224,15 +209,15 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if puts.Ended(p.Step) {
if delivery.PutEnded(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), puts.Released, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := puts.Encode(p)
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
if err != nil {
return nil, err
}
@@ -240,7 +225,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return answerOf(node, module, p), nil
return putAnswer{Node: node, Module: module, Put: p}, nil
}
// doPuts is the tool assigner_puts.
@@ -251,10 +236,8 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
}
sort.Strings(keys)
out := []putAnswer{}
newest := map[string]string{} // the indexer's newest fingerprint by module, asked once and only for a put that stands
asked := false
for _, key := range keys {
node, module, err := puts.ParseKey(key)
node, module, err := delivery.ParsePutKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
@@ -262,27 +245,11 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
p, err := puts.Decode(raw)
p, err := delivery.DecodePut(raw)
if err != nil {
continue
}
a := answerOf(node, module, p)
if !puts.Ended(p.Step) {
if !asked {
asked = true
if mods, err := indexerView(s.ask, indexerModules); err == nil {
for name, raw := range mods {
if m, err := meshcatalogclient.DecodeModule(raw); err == nil {
newest[name] = m.Fp2
}
}
}
}
if n := newest[module]; n != "" && n != p.FP {
a.Older = n
}
}
out = append(out, a)
out = append(out, putAnswer{Node: node, Module: module, Put: p})
}
return map[string]any{"puts": out}, nil
}
@@ -339,12 +306,9 @@ func (s *putTools) tools() []stdio.Tool {
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
"failed (the failing check is the delivery's), or rejudge (judge it again now); written in the hand-act log first"},
[]string{"module", "why"}),
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
[]string{"node", "module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
s.waitsTool(),
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
@@ -372,8 +336,7 @@ func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(st
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
func serveTools() error {
plc.SetModule(os.Getenv(stdio.ServedAs)) // the module the runtime serves these tools as (A-296)
s := newPutTools(runtimeStore{}, stdio.Ask)
s := newPutTools(runtimeStore{})
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
@@ -388,44 +351,19 @@ func serveTools() error {
time.Sleep(wait)
}
}()
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
gate := seatlist.NewPart("assigner")
go func() {
<-stdio.Served()
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
}()
seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask)
seatVerbs.seatList = gate
tools := append(s.tools(), seatVerbs.tools()...)
tools = append(tools, gate.Tools()...)
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
return stdio.SeatPublish(wantsEventSubject(e), b, id)
})
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
seatVerbs.watched = wants.watched
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
tools = append(tools, wants.tools()...)
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// What each node's engine reads, as the assigner uses it: its own state engine-reads (design 47 H-A-104).
tools = append(tools, engineReadsTools(runtimeStore{})...)
tools = append(tools, writtenTools(runtimeStore{})...)
tools = append(tools, whyTools(runtimeStore{})...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777)
tools = append(tools, keepNowTools(stdio.Ask)...)
tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2))
return stdio.Serve("", tools)
}
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := catalogkey.KeyPart(fp)
part, err := delivery.FingerprintKeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
@@ -1,493 +0,0 @@
package main
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/buckets"
)
// The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat
// mesh-wants, asked by the runner for each mover before its composition step. One run per take-in revision, a key
// `<pipeline>.validate` of the state composition-runs in the CompositionRun form with no machine, which every mover's step
// reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from
// the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the
// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input.
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case
// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed.
const (
// validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's
// cmd/mesh-judge, design 47 H-A-34).
validatorPackage = "mesh-delivery-indexer/cmd/mesh-judge"
// validateKeySuffix ends the key of a revision's validate run, `<pipeline>.validate` (H-A-108).
validateKeySuffix = "." + meshwants.StepRunStepValidate
// beforeCase is the validator's case of the revision's base, beside changeCase (H-A-108 (3), H-A-151, H-A-154).
beforeCase = "before"
// The reasons a validate step fails with, the first word of its said (§745).
reasonManifestRefused = "manifest-refused"
reasonCycle = "cycle"
// betweenManifests is the validator's name for what it judges between manifests (the catalogue as a whole).
betweenManifests = "(between manifests)"
)
// validateKeyOf is the key of the validate run of the step's take-in revision.
func validateKeyOf(step meshwants.StepRun) (string, error) {
if step.Pipeline == "" {
return "", errors.New("the step names no pipeline, so no validate run is named")
}
return step.Pipeline + validateKeySuffix, nil
}
// isValidateRun says whether a run of the state is a revision's validate run: one with no machine.
func isValidateRun(run meshwants.CompositionRun) bool { return run.Machine == "" }
// validate runs one mover's validate step: it joins or starts the revision's validate run and judges the step from its end.
func (r *compositionRun) validate(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
key, err := validateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
j, err := r.runs.joinAt(key, readerOf(id, step), candidate, "", buildstep.NewID())
if err != nil {
return couldNotRun("the validate run " + key + " is not joined: " + err.Error())
}
if j.started {
go r.startValidate(ctx, key, j.run, step)
}
return r.readValidate(ctx, key, step)
}
// resumeValidate judges a validate step taken up at the assigner's start from its revision's run.
func (r *compositionRun) resumeValidate(ctx context.Context, _ string, step meshwants.StepRun) stepOutcome {
key, err := validateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
return r.readValidate(ctx, key, step)
}
// readValidate waits for the end of the validate run of key and judges the step from it.
func (r *compositionRun) readValidate(ctx context.Context, key string, step meshwants.StepRun) stepOutcome {
setShared(ctx, r.runs.sharedOf([]string{key}, nil))
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
run := r.runs.await(ctx, []string{key})[key]
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
out := r.judgeValidate(ctx, step, run)
out.Shared = r.runs.sharedOf([]string{key}, map[string]meshwants.CompositionRun{key: run})
return out
}
// judgeValidate is a step's end from its revision's validate run: could not run with the run's words; failed `cycle` naming
// the cycle; failed `manifest-refused` naming every problem the revision causes; else passed.
func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepRun, run meshwants.CompositionRun) stepOutcome {
if run.Ended == nil {
return couldNotRun("the step stopped before its validate run ended")
}
if run.Outcome != meshwants.CompositionRunOutcomePassed {
return couldNotRun(run.Said)
}
var v validateVerdict
if raw, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(raw, &v) != nil {
return couldNotRun("the validate run's verdict does not read")
}
if len(v.Cycle) > 0 {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: reasonCycle + ": " + cycleWords(v.Cycle)}
}
half, err := r.revision(ctx, step)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
// A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151).
empty := ""
if half.Revision.Base <= 0 {
empty = emptyBeforeWords
}
if problems := causedProblems(v.Change, v.Before); len(problems) > 0 {
said := reasonManifestRefused + ": " + strings.Join(problems, "; ")
if empty != "" {
said += " (" + empty + ")"
}
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: said}
}
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed, Said: empty}
}
// emptyBeforeWords say, in a validate step's end, that its revision names no base: the branch's first take-in, judged against
// an empty case before (H-A-151, H-A-154).
const emptyBeforeWords = "the case before is empty: the revision names no base, the branch's first take-in"
// validateVerdict is a validate run's verdict as the run keeps it: the cycle the revision makes, or the validator's two cases.
type validateVerdict struct {
Cycle []meshcatalog.ArtifactRef `json:"cycle,omitempty"`
Change composer.CaseVerdict `json:"change"`
Before composer.CaseVerdict `json:"before"`
}
// asVerdict is v as a run's verdict.
func (v validateVerdict) asVerdict() (map[string]any, error) {
raw, err := json.Marshal(v)
if err != nil {
return nil, err
}
var out map[string]any
return out, json.Unmarshal(raw, &out)
}
// cycleWords names a cycle as the artifacts it passes through.
func cycleWords(cycle []meshcatalog.ArtifactRef) string {
words := make([]string, len(cycle))
for i, a := range cycle {
words[i] = string(a)
}
return strings.Join(words, " → ")
}
// before is the case before of the step's revision half: the revision its base names, the newest kept revision below it that
// its take-in pinned, read through the seat mesh-catalog's verb revision (H-A-108 (3), H-A-151, H-A-154); empty where the
// revision names no base, the branch's first take-in.
func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, half mcc.Half) (mcc.Half, error) {
if half.Revision.Base <= 0 {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}, nil
}
base := step
base.Revision = half.Revision.Base
return r.revision(ctx, base)
}
// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between
// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the
// revision alters that manifest or not (assigner_validate_before answers those).
func causedProblems(change, before composer.CaseVerdict) []string {
return problemsBy(change, before, false)
}
// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814).
func uncountedProblems(change, before composer.CaseVerdict) []string {
return problemsBy(change, before, true)
}
// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had).
func problemsBy(change, before composer.CaseVerdict, had bool) []string {
var out []string
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
read := change.Manifests[module]
if read.Accepted || read.Refused == "" {
continue
}
held := map[string]bool{}
for _, p := range strings.Split(before.Manifests[module].Refused, "\n") {
held[p] = true
}
var named []string
for _, p := range strings.Split(read.Refused, "\n") {
if p != "" && held[p] == had {
named = append(named, p)
}
}
if len(named) == 0 {
continue
}
what := module
if module == betweenManifests {
what = "between manifests"
}
out = append(out, what+": "+strings.Join(named, "; "))
}
return out
}
// catalogEntry is the entry of the module that holds the seat mesh-catalog, or false.
func catalogEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
for _, name := range slices.Sorted(maps.Keys(entries)) {
for _, s := range entries[name].Serves {
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), meshcatalog.Seat+".") {
return entries[name], true
}
}
}
return meshcatalog.CatalogEntry{}, false
}
// manifestsOf are the half's manifests whole, as the validator's case holds them.
func manifestsOf(half mcc.Half) (map[string]composer.Manifest, error) {
out := map[string]composer.Manifest{}
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
out[name] = composer.Manifest{Raw: raw}
}
return out, nil
}
// startValidate runs the validate run the step started: the revision's cycle ends it at once; else the validator's request
// is kept once in `inputs`, its program and input recorded on the run, and one run ask put on the Go build seat. A fault ends
// the run could not run.
func (r *compositionRun) startValidate(ctx context.Context, key string, run meshwants.CompositionRun, step meshwants.StepRun) {
ask := run.Ask
end := func(outcome, said string, verdict map[string]any) {
if _, err := r.runs.endRun(key, ask, outcome, said, verdict); err != nil {
r.logf("the validate run %s is not ended: %v", key, err)
}
}
couldNot := func(said string) { end(meshwants.CompositionRunOutcomeCouldNotRun, said, nil) }
half, err := r.revision(ctx, step)
if err != nil {
couldNot("the step's catalog revision is not read: " + err.Error())
return
}
if cycle := graph.RefusedCycle(half.Entries); cycle != nil {
verdict, err := validateVerdict{Cycle: cycle}.asVerdict()
if err != nil {
couldNot(err.Error())
return
}
end(meshwants.CompositionRunOutcomePassed, "", verdict)
return
}
before, err := r.before(ctx, step, half)
if err != nil {
couldNot("the base of the case before is not read: " + err.Error())
return
}
entry, ok := catalogEntry(half.Entries)
if !ok {
couldNot(fmt.Sprintf("the catalog revision %d holds no module claiming the seat %s, so no manifest validator is built", half.Revision.Revision, meshcatalog.Seat))
return
}
program, err := programOfEntry(entry)
if err != nil {
couldNot(err.Error())
return
}
program.Package = validatorPackage
seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions)
if err != nil {
couldNot(err.Error())
return
}
change, err := manifestsOf(half)
if err != nil {
couldNot(err.Error())
return
}
prior, err := manifestsOf(before)
if err != nil {
couldNot(err.Error())
return
}
// No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)).
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}},
Seats: seats, SeatList: list}
raw, err := json.Marshal(req)
if err != nil {
couldNot("its request cannot be encoded: " + err.Error())
return
}
digest, err := r.keeper.keep(ask, raw)
if err != nil {
couldNot("its request is not kept: " + err.Error())
return
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
couldNot(err.Error())
return
}
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: entry.Fp}
c.Input = digest
return true
}); err != nil {
couldNot("the run's program and input are not written: " + err.Error())
return
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the manifest validator of " + step.Pipeline + " at " + program.Commit},
Packages: pinsOfEntry(entry)} // the validator is built against its entry's pins at the step's revision (H-A-203, H-A-192, H-A-205)
if err := r.put(ctx, a); err != nil {
couldNot("its run ask " + ask + " is not put: " + err.Error())
}
}
// validateRunVerdict reads the validator's answer: its two cases as the run keeps them, or why it is none.
func validateRunVerdict(answer []byte) (map[string]any, string) {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return nil, "the manifest validator answered no verdict: " + err.Error()
}
if v.V != composer.Version {
return nil, fmt.Sprintf("the manifest validator answered a verdict of version %d, the assigner reads %d", v.V, composer.Version)
}
var out validateVerdict
found := 0
for _, c := range v.Cases {
switch c.Name {
case changeCase:
out.Change, found = c, found+1
case beforeCase:
out.Before, found = c, found+1
}
}
if found != 2 {
return nil, "the manifest validator's verdict does not hold both the cases " + changeCase + " and " + beforeCase
}
verdict, err := out.asVerdict()
if err != nil {
return nil, "the manifest validator's verdict cannot be kept: " + err.Error()
}
return verdict, ""
}
// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814).
const validateBeforeTool = "assigner_validate_before"
// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too,
// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps.
func validateBefore(st store, pipeline, module string) (map[string]any, error) {
key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)})
if err != nil {
return nil, err
}
raw, _, ok, err := st.get(buckets.CompositionRunsState, key)
if err != nil {
return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err)
}
if !ok {
return nil, fmt.Errorf("no validate run %s is kept", key)
}
var run meshwants.CompositionRun
if err := json.Unmarshal(raw, &run); err != nil {
return nil, fmt.Errorf("the validate run %s does not read: %w", key, err)
}
var v validateVerdict
if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil {
return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key)
}
change, before := v.Change, v.Before
if module = strings.TrimSpace(module); module != "" {
change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}}
}
uncounted := uncountedProblems(change, before)
if uncounted == nil {
uncounted = []string{}
}
return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil
}
// validateBeforeTools are the module's tool of the validate step's before case.
func validateBeforeTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " +
"validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " +
"for one module when named. Read-only.",
Input: schema(map[string]string{"pipeline": "the pipeline's key, <branch>.<revision>", "module": "one module; absent: every one"}, []string{"pipeline"}),
Run: func(args map[string]any) (any, error) {
return validateBefore(st, arg(args, "pipeline"), arg(args, "module"))
}}}
}
// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443).
const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList
// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by
// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them,
// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb
// seat-list hands, asked as a kept call under the run's ask.
func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) {
if len(defs) == 0 {
return r.seatList(ask)
}
sorted := slices.Clone(defs)
slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) })
seats, err := r.seats(ctx, sorted)
if err != nil {
return nil, "", err
}
raw, err := json.Marshal(seats)
if err != nil {
return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err)
}
var list []json.RawMessage
if err := json.Unmarshal(raw, &list); err != nil {
return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err)
}
return list, seatListDigest(raw), nil
}
// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex.
func seatListDigest(raw []byte) string {
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in
// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest
// judges the same bytes.
func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) {
what := "the trunk's seat list is not read through " + seatListKey
if r.callFile == nil {
return nil, "", errors.New(what + ": the assigner holds no chunked call")
}
var bits [16]byte
if _, err := rand.Read(bits[:]); err != nil {
return nil, "", fmt.Errorf("%s: %w", what, err)
}
raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey,
meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])})
if err != nil {
return nil, "", fmt.Errorf("%s: %w", what, err)
}
var a struct {
meshcatalog.SeatListAnswer
File string `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err)
}
if a.Digest == "" {
return nil, "", errors.New(what + ": its answer names no digest")
}
file, err := base64.StdEncoding.DecodeString(a.File)
if err != nil {
return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err)
}
if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest {
return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file),
seatListDigest(file), a.Size, a.Digest)
}
var list []json.RawMessage
if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 {
return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest)
}
return list, a.Digest, nil
}
@@ -1,220 +0,0 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"slices"
"strings"
"sync"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/buckets"
)
// A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate
// run, `<pipeline>.validate` of the state composition-runs; a manifest the revision alters that names an unknown field fails
// it `manifest-refused` naming the problem, a problem of a manifest it leaves as it was does not, and the validator is the
// revision's own entry of the module holding the seat mesh-catalog, on the Go build seat; a revision whose entries make a cycle
// fails it `cycle` naming the cycle, with no validator asked. The case before is the revision its base names (H-A-151, H-A-154:
// the newest kept below it, which the stand-in names as the indexer does); a revision naming no base, the branch's first take-in,
// is judged against an empty case before, said in its end, and no revision below it is asked.
func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer",
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix",
Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient": "v0.1.4"}}
entry := func(module, digest string, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges}
}
revisions := map[int64][]meshcatalog.CatalogEntry{
6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")},
7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")},
3: {indexer},
12: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
}
// base is the newest kept revision below n, as the take-in names it; none below the first. Revision 12 names 7 with 8 to 11
// never kept, so a case before read as the number minus one could not run (H-A-154).
base := func(n int64) (b int64) {
if n == 12 {
return 7
}
for k := range revisions {
if k < n && k > b {
b = k
}
}
return b
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
switch key {
case mcc.RevisionVerb:
var in meshcatalog.RevisionInput
_ = json.Unmarshal(raw, &in)
entries, ok := revisions[in.Revision]
if !ok || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
// Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)).
var defs []meshcatalog.CatalogRevisionDefinitionsItem
if in.Revision != 3 {
defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}
}
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision),
Definitions: defs}, Entries: entries})
case seatDefinitionsKey:
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}})
}
return nil, errors.New("no such verb " + key)
}
// The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file.
trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`)
listAnswer := func(file []byte, digest string, size int) json.RawMessage {
raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)})
return raw
}
var listAsks []process.KeptAsk
listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
in, _ := body.(meshcatalog.SeatListInput)
if key != seatListKey || len(in.Call) != 32 {
return nil, errors.New("asked " + key + " with the call " + in.Call)
}
listAsks = append(listAsks, ask)
return listed, nil
}
var mu sync.Mutex
var asks []buildstep.StepAsk
var requests []composer.Request
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf,
put: func(_ context.Context, a buildstep.StepAsk) error {
body, err := keeper.body(a.Run.Input.Digest)
if err != nil {
return err
}
req, err := composer.Decode(body)
if err != nil {
return err
}
mu.Lock()
asks, requests = append(asks, a), append(requests, req)
mu.Unlock()
answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{
{Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}},
{Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}})
go func() {
if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"},
func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil {
t.Error(err)
}
}()
return nil
}}
steps := &compositionSteps{validate: r.validate, run: r.run}
if steps.howOf(meshwants.StepRunStepValidate, false) == nil || steps.howOf("check", false) != nil {
t.Error("the validate step is not taken, or another step is")
}
step := func(module string, revision int64) meshwants.StepRun {
return meshwants.StepRun{Step: meshwants.StepRunStepValidate, Pipeline: "main." + string(rune('0'+revision)), Branch: "main", Module: module,
Revision: revision, Source: revision}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
outs := make([]stepOutcome, 2)
var wg sync.WaitGroup
for i, module := range []string{"web", "db"} {
wg.Add(1)
go func() { defer wg.Done(); outs[i] = r.validate(ctx, "ask-"+module, step(module, 7)) }()
}
wg.Wait()
for i, o := range outs {
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) ||
strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems
t.Errorf("mover %d's validate step ended %+v", i, o)
}
}
// assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count.
runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}}
kept, _, _, _ := runs.read("main.7" + validateKeySuffix)
runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept)
before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"})
if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) {
t.Errorf("assigner_validate_before answered %+v, %v", before, err)
}
if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil {
t.Error("assigner_validate_before answers a run not kept")
}
if len(asks) != 1 {
t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks))
}
if p := asks[0].Run.Program; p.Commit != commitE3 || p.Dir != "modules/mesh-delivery-indexer" || p.Package != validatorPackage || len(asks[0].Run.Heads) != 0 {
t.Errorf("the validator is built from %+v", asks[0].Run)
}
// The validator is built against the indexer entry's pins at the step's revision (H-A-203, H-A-192, H-A-205).
if got := asks[0].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"] != "v0.1.4" {
t.Errorf("the validator is built against the pins %v; want its entry's", got)
}
if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil {
t.Errorf("the validator's request is %+v", requests[0])
}
// A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat
// list is asked.
if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 {
t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own))
}
if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok {
t.Error("the validate run is not the key <pipeline>.validate of composition-runs")
}
o := r.validate(ctx, "ask-a", step("a", 9))
if o.Outcome != meshwants.StepEndedOutcomeFailed || o.Said != reasonCycle+": a:x → b:y → a:x" || len(asks) != 1 {
t.Errorf("a revision making a cycle ended %+v (%d validator asks)", o, len(asks))
}
// The stand-in validator answers its one verdict: judged against the empty case before, web's problem is the revision's.
if o := r.validate(ctx, "ask-first", step("indexer", 3)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 {
t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks))
}
// Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest,
// asked once as a kept call under the run's ask (H-A-441 (b), H-A-443).
if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) ||
len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 {
t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList))
}
// Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes.
for _, row := range []struct {
name string
answer json.RawMessage
}{
{"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))},
{"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)},
{"no digest", listAnswer(trunkList, "", len(trunkList))},
{"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))},
{"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)},
} {
listed = row.answer
if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) {
t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err)
}
}
listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
// Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one
// is not, and the case before holds base 7's manifests.
if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
o.Said != reasonManifestRefused+`: web: unknown field "colour"` || len(asks) != 3 {
t.Errorf("a revision naming base 7 below its number ended %+v (%d validator asks), want it judged against base 7", o, len(asks))
} else if web := requests[2].Cases[1].Manifests["web"]; len(requests[2].Cases[1].Manifests) != 3 || !strings.Contains(string(web.Raw), "d-web-2") {
t.Errorf("the case before of revision 12 is %+v, want base 7's manifests", requests[2].Cases[1].Manifests)
}
}
@@ -1,149 +0,0 @@
package main
import (
"slices"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
// manifests, as a provider is counted (ADR 0603 §5).
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
// a verb.
func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
if len(g) == 0 {
return nil
}
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
func(module string) []string { return g[module].Serves })
if p.unread.changed(unread) && len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
}
return served
}
// saidSet is the last set of names said, so a pass says a set again only when it changed.
type saidSet struct {
mu sync.Mutex
last string
}
// changed keeps names as the set said last and says whether it differs from the one before.
func (s *saidSet) changed(names []string) bool {
sorted := slices.Sorted(slices.Values(names))
key := strings.Join(sorted, " ")
s.mu.Lock()
defer s.mu.Unlock()
if key == s.last {
return false
}
s.last = key
return true
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
// that build serves what it offers (A-352); a consumer needing the newer build's forms waits by its kind-4 edge instead. A
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
// holds a consumer for good.
func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
offered := map[string]bool{}
for node, haves := range n.Haves {
if !n.Online[node] {
continue
}
for module, h := range haves {
if h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseWaitingYes || h.Health != nodeengineclient.HealthOK {
continue
}
m, ok := p.manifestOf(module, h.Fp)
if !ok {
m = trunk[module]
}
for _, o := range m.Offers() {
offered[o] = true
}
}
}
return func(provision string) bool { return offered[provision] }
}
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
func (p *passer) servesOf(module, fp string) ([]string, bool) {
m, ok := p.manifestOf(module, fp)
if !ok {
return nil, false
}
return graph.ServesOf(m), true
}
// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests
// hold), and whether it could be read.
func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
e, ok := p.w.ends.endOf(module, fp)
if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" {
return catalogue.Manifest{}, false
}
got := p.w.manifest(e.Manifest)
if got.err != "" {
return catalogue.Manifest{}, false
}
return got.manifest, true
}
// servingHaves are the haves rule 8 counts (design 47 G A-426, A-426a): a node that runs a healthy build of a part serves what that
// build serves, whatever its version and whether or not the catalog lists it, so a have held (its build has no catalog entry, a
// put holds it) or waiting for a person's yes, whose build runs healthy, counts as running; a running statement that names a build
// stopped, or not healthy, does not. counted is told each build so counted that its phase alone would have left out.
func servingHaves(n assign.Nodes, counted func(node, module string, h nodeengine.HaveByNodeModuleValue)) map[string]map[string]nodeengine.HaveByNodeModuleValue {
out := map[string]map[string]nodeengine.HaveByNodeModuleValue{}
for node, haves := range n.Haves {
out[node] = map[string]nodeengine.HaveByNodeModuleValue{}
for module, h := range haves {
if (h.Phase == nodeengineclient.PhaseHeld || h.Phase == nodeengineclient.PhaseWaitingYes) && h.Health == nodeengineclient.HealthOK {
if r, ok := n.Running[node][module]; !ok || (!r.Stopped && (r.Healthy == nil || *r.Healthy)) {
if counted != nil {
counted(node, module, h)
}
h.Phase = nodeengineclient.PhaseRunning
}
}
out[node][module] = h
}
}
return out
}
// countedServing says once per node, module and build that rule 8 counted a build whose have is not in phase running as serving
// its verbs (A-426a): the rule says which build it counted.
func (p *passer) countedServing(node, module string, h nodeengine.HaveByNodeModuleValue) {
key := "serving " + node + "." + module + "@" + h.Fp + "." + h.Phase
if p.skipped[key] {
return
}
if p.skipped == nil {
p.skipped = map[string]bool{}
}
p.skipped[key] = true
p.logf("%s on %s runs %s healthy in phase %s: counted as serving what that build serves, whatever its version (design 47 G A-426a)", module, node, shortHex(h.Fp), h.Phase)
}
@@ -1,196 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// marksBucket is the assigner's state of what its views family said each state up to.
func marksBucket() string { return assignment.Module() + "_" + views.MarksState }
// viewedTool is one tool over one state of the assigner's.
type viewedTool struct{ tool, state, words string }
// firstViewTools are the assigner's first-view tools (ADR 0621 §1): each answers only its callers list
// (views.CallersEnv), which the manifest writes.
var firstViewTools = []viewedTool{
{"assignments", mma.StateAssignments, "where each module is placed: the seat mesh-module-assigner's assignments, by module."},
{"seat-holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"wants", "wants", "what each node should run: want.<node>.<module>, its fingerprint and the rule that gave it."},
// The seat mesh-wants' asks (design 47 G §717): the asks the assigner put to a person about what a node runs.
{"asks", "asks", "the asks the assigner put to a person about what a node runs, and their answers."},
}
// readTools are the assigner's ordinary read tools, one per state its readers read (design 25 § 12.2).
var readTools = []viewedTool{
{"assigner_read_assignments", mma.StateAssignments, "where each module is placed, by module."},
{"assigner_read_seat_holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
{"assigner_read_findings", assign.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
{"assigner_read_copies", copiesState, "the copies the assigner made of a retired state: when the copy into assignments ended and how many keys it copied."},
}
// viewedStates are the states the views family says, each by its bucket: the six its readers read.
func viewedStates() map[string]string {
return map[string]string{
mma.StateAssignments: plc.AssignmentsBucket(),
mma.StateSeatHolders: plc.SeatHoldersBucket(),
"wants": buckets.Wants(),
assign.FindingsState: assign.FindingsBucket(),
"asks": buckets.Asks(),
"differences": buckets.Differences(),
copiesState: copiesBucket(),
}
}
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
// runtime of its own is connected (ADR 0621 §3).
func viewTools(getenv func(string) string, state func(string) views.Source, now func() time.Time) []stdio.Tool {
marks := state(views.MarksState)
var out []stdio.Tool
for _, t := range firstViewTools {
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
}
for _, t := range readTools {
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
}
// Its own first views, of the indexer's manifests, modules and graph (DLV-F3r, ADR 0631 §2): read, and asked again by hand.
return append(out, views.StatusTools(assignment.Module(), "assigner_first_views", "assigner_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
}
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, writer views.WireWriter, errs chan<- error, logf func(string, ...any)) {
process.AwaitBucket(ctx, js, nc, marksBucket(), "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
em := viewsEmitter(js, account, mk, writer, logf)
for _, state := range []string{"asks", "differences", assign.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
state := state
start := func(kv jetstream.KeyValue) {
if err := em.Start(ctx, state, kv, errs); err != nil {
logf("%v: its readers ask its first view at each change until the assigner starts again", err)
}
}
if kv := kvs[viewedStates()[state]]; kv != nil {
start(kv)
continue
}
process.AwaitBucket(ctx, js, nc, viewedStates()[state], "its changes are not said on the views family", logf).Ready(start)
}
})
}
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
// view-unsaid, in its findings, written through its finding writer (review round 2); with none it is said in the log only.
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: assignment.Module(), Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logf}
}
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
func firstViewsBucket() string { return assignment.Module() + "_" + views.StatusState }
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195, A-26): the
// module's views reader adopts each branch's follower, so it asks the seat mesh-catalog's verb `revision` (the trunk's, then
// each branch a registration names), follows its event `catalog-revision`, asks once more on a gap, and past the bound raises
// the finding naming the seat and the verb, cleared by the answer, in the assigner's findings. Its status is read by
// assigner_first_views and asked again by assigner_first_view_again.
type catalogViews struct {
js jetstream.JetStream
account process.Account
call func(string, any) (json.RawMessage, error)
writer views.WireWriter
status *process.AwaitedBucket
logf func(string, ...any)
hub *process.EventHub
h *halves
// candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made;
// before that a candidate's revision is said not taken.
candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error]
// answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers,
// then its views family; nil when it is not followed.
answers *views.Picture
}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
// seconds). A test shortens them.
var firstViewBound, firstViewWait time.Duration
// start adopts the trunk's catalog half through the reader and starts the reader; it answers how many settles to wait for.
func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
reader := &views.ProcessReader{Module: assignment.Module(), Call: iv.call, Logf: iv.logf, Hub: iv.hub, Bound: firstViewBound, Wait: firstViewWait}
if iv.writer != nil {
reader.Findings = func() views.WireWriter { return iv.writer }
}
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
if err := iv.h.follow(); err != nil {
iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err)
}
// Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185).
if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
if f := iv.candidates.Load(); f != nil {
return (*f)(ctx, ev)
}
return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision)
}); err != nil {
iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err)
}
// The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket.
if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil {
iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err)
} else {
iv.answers = pic
}
reader.Start(ctx, iv.js, iv.account, iv.status)
return 1
}
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
// findings state it is said in the log alone.
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
w := heard.Writer{Observer: assignment.Module(), Logf: logf}
if findings.Put != nil {
w.Put = func(_ string, value []byte) error {
var f mcseat.Finding
if err := json.Unmarshal(value, &f); err != nil {
return err
}
return findings.Put(f)
}
}
if findings.Delete != nil {
w.Delete = func(key string) error {
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
return nil // none stands
}
return findings.Delete(key)
}
}
return w
}
@@ -1,146 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// wantDropTool drops one want of a node for a module by hand (design 47 H §765, H-A-58): a held or failed want of the earlier
// model goes at the clean state, its key deleted and the act written in the hand-act log; the next pass writes the want of the
// new form.
const wantDropTool = "assigner_want_drop"
// wantsState is the state the wants are kept in.
const wantsState = "wants"
// dropWant deletes the want of module on node, the act first recorded through the seat mesh-hand-acts.
func dropWant(st seatStore, ask func(string, any) (json.RawMessage, error), node, module, why, by string) (any, error) {
switch {
case node == "" || module == "":
return nil, errors.New("name the node and the module whose want is dropped")
case why == "":
return nil, errors.New("a want dropped by hand says why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
}
key, err := mwc.WantKey(node, module)
if err != nil {
return nil, err
}
raw, _, found, err := st.get(wantsState, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the assigner holds no want under %s", key)
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
if err != nil {
return nil, fmt.Errorf("the drop of %s is not recorded, so the want stays: %w", key, err)
}
if err := st.del(wantsState, key); err != nil {
return nil, fmt.Errorf("the want %s is not dropped: %w", key, err)
}
return map[string]any{"key": key, "dropped": json.RawMessage(raw), "by": by, "hand-act": taken.Log}, nil
}
// The kinds the all form counts what it dropped by (design 47 H-A-123).
const (
dropHeld = "held"
dropCurrent = "current"
dropUndecodable = "undecodable"
)
// dropAllWants deletes every want key of the state wants, held or not, and every value of an earlier form the decoder refuses
// (the all form, design 47 H-A-121 (2), H-A-123): each act recorded through the seat mesh-hand-acts first, as the one-item form
// does; it answers what it dropped, counted per kind. A drop not recorded stops it, saying what went before.
func dropAllWants(st seatStore, ask func(string, any) (json.RawMessage, error), why, by string) (any, error) {
switch {
case why == "":
return nil, errors.New("a want dropped by hand says why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
}
all, err := st.keys(wantsState)
if err != nil {
return nil, err
}
slices.Sort(all)
counted := map[string]int{dropHeld: 0, dropCurrent: 0, dropUndecodable: 0}
dropped := map[string][]string{}
answer := func() map[string]any {
return map[string]any{"dropped": dropped, "counted": counted, "by": by}
}
for _, key := range all {
if !strings.HasPrefix(key, "want.") {
continue
}
raw, _, found, err := st.get(wantsState, key)
if err != nil {
return answer(), fmt.Errorf("the want %s cannot be read, so the drop stops there: %w", key, err)
}
if !found {
continue
}
kind := wantKind(key, raw)
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
if err != nil {
return answer(), fmt.Errorf("the drop of %s is not recorded, so it and the wants after it stay: %w", key, err)
}
if err := st.del(wantsState, key); err != nil {
return answer(), fmt.Errorf("the want %s is not dropped: %w", key, err)
}
counted[kind]++
dropped[kind] = append(dropped[kind], key+" (hand act "+taken.Log+")")
}
return answer(), nil
}
// wantKind says how the all form counts a want: undecodable when its key or value is of no form the decoder takes, held when it
// holds, current otherwise (design 47 H-A-123).
func wantKind(key string, raw []byte) string {
if _, err := mwc.ParseWantNodeKey(key); err == nil {
if _, err := mwc.DecodeNodeWant(raw); err != nil {
return dropUndecodable
}
return dropCurrent
}
if _, _, err := mwc.ParseWantKey(key); err != nil {
return dropUndecodable
}
w, err := mwc.DecodeWant(raw)
switch {
case err != nil:
return dropUndecodable
case w.Hold:
return dropHeld
}
return dropCurrent
}
// wantDropTools is the tool over the runtime's states.
func wantDropTools(st seatStore, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: wantDropTool, Description: "Drop one want by hand (design 47 H §765): the want of module on node is " +
"deleted from the state wants, as the clean state drops a held or failed want of the earlier model; the next pass writes the " +
"want of the new form. Recorded as a hand act first; refused with no want under the key. With all true and no node or module, " +
"every want key goes, held or not, and every value of an earlier form the decoder refuses, each recorded as a hand act; it " +
"answers what it dropped, counted per kind (held, current, undecodable).",
Input: schema(map[string]string{"node": "the node's name", "module": "the module's name", "why": "why the want is dropped",
"all": "true drops every want of the state wants, naming no node or module"}, []string{"why"}, "all"),
Run: func(a map[string]any) (any, error) {
if arg(a, "all") == "true" {
if arg(a, "node") != "" || arg(a, "module") != "" {
return nil, errors.New("the all form drops every want: name no node or module with it")
}
return dropAllWants(st, ask, arg(a, "why"), stdio.CallerOf(a))
}
return dropWant(st, ask, arg(a, "node"), arg(a, "module"), arg(a, "why"), stdio.CallerOf(a))
}}}
}
@@ -1,99 +0,0 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
)
// A12/AC1 (design 47 H §765, H-A-58): assigner_want_drop with node, module and why records the act in the hand-act log first
// and then deletes that want; with no want under the key, no why or no proven caller, and when the act is not recorded, nothing is
// dropped.
func TestAWantIsDroppedByHandAndRecorded(t *testing.T) {
key, _ := mwc.WantKey("ace", "web")
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
for _, c := range []struct {
name, node, module, why, by string
refuse error
dropped bool
}{
{"dropped", "ace", "web", "the clean state", "person.jochen", nil, true},
{"no want there", "ace", "db", "the clean state", "person.jochen", nil, false},
{"no why", "ace", "web", "", "person.jochen", nil, false},
{"no caller", "ace", "web", "the clean state", "", nil, false},
{"not recorded", "ace", "web", "the clean state", "person.jochen", errors.New("no responders"), false},
} {
st := memStore{wantsState: &memKV{values: map[string][]byte{key: held}}}
var asked []string
ask := func(verb string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, verb+" "+string(raw))
return json.RawMessage(`{}`), c.refuse
}
out, err := dropWant(st, ask, c.node, c.module, c.why, c.by)
_, there := st[wantsState].values[key]
switch {
case c.dropped && (err != nil || there || len(asked) != 1 || !strings.Contains(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], wantDropTool+" "+key)):
t.Errorf("%s: %v, %v; the want left %v, asked %v", c.name, out, err, there, asked)
case !c.dropped && (err == nil || !there):
t.Errorf("%s: %v, %v; the want left %v, want it refused and kept", c.name, out, err, there)
}
}
}
// H-A-121 (2), H-A-123: assigner_want_drop's all form drops every want key, held or not, and every value of an earlier form,
// each recorded as a hand act first, answering the count per kind; a held fingerprint's key stays; a drop not recorded stops it;
// the all form names no node or module.
func TestEveryWantIsDroppedByHandAndCounted(t *testing.T) {
heldKey, _ := mwc.WantKey("ace", "web")
curKey, _ := mwc.WantKey("ace", "db")
nodeKey, _ := mwc.WantNodeKey("ace")
oldKey, _ := mwc.WantKey("g14", "web")
fpKey, _ := mwc.HeldKey("web", fpTest)
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
cur, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest})
node, _ := mwc.EncodeNodeWant(mwc.NodeWant{Self: true})
values := func() map[string][]byte {
return map[string][]byte{heldKey: held, curKey: cur, nodeKey: node, oldKey: []byte(`{"fingerprint":"old"}`), fpKey: []byte(`{}`)}
}
for _, c := range []struct {
name string
refuse error
left int
}{
{"dropped", nil, 1},
{"not recorded", errors.New("no responders"), 5},
} {
st := memStore{wantsState: &memKV{values: values()}}
var asked []string
ask := func(verb string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, string(raw))
return json.RawMessage(`{}`), c.refuse
}
out, err := dropAllWants(st, ask, "the clean state", "person.jochen")
if left := len(st[wantsState].values); left != c.left || (c.refuse == nil) != (err == nil) {
t.Errorf("%s: %v, %v; %d keys left, want %d", c.name, out, err, left, c.left)
}
if c.refuse != nil {
continue
}
if _, ok := st[wantsState].values[fpKey]; !ok {
t.Errorf("the held fingerprint's key %s is dropped: it is no want", fpKey)
}
counted := out.(map[string]any)["counted"].(map[string]int)
if counted[dropHeld] != 1 || counted[dropCurrent] != 2 || counted[dropUndecodable] != 1 || len(asked) != 4 {
t.Errorf("counted %v, asked %d hand acts; want held 1, current 2, undecodable 1 and 4 acts", counted, len(asked))
}
}
if _, err := dropAllWants(memStore{wantsState: &memKV{values: values()}}, nil, "", "person.jochen"); err == nil {
t.Error("the all form dropped with no why")
}
run := wantDropTools(memStore{wantsState: &memKV{values: values()}}, nil)[0].Run
if _, err := run(map[string]any{"all": "true", "node": "ace", "why": "x"}); err == nil || !strings.Contains(err.Error(), "name no node") {
t.Errorf("the all form with a node: %v, want it refused", err)
}
}
@@ -4,35 +4,31 @@ import (
"encoding/json"
"errors"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strconv"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-module-assigner's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
const (
policiesState = buckets.PoliciesState
policiesState = delivery.AssignerPoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
composedState = "mesh-controller.composed"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
@@ -49,20 +45,24 @@ type wantsTools struct {
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
// a large state (issue 1953 L5f). Nil reads key by key.
watched *stdio.WatchedStates
// revisions is the kept data assignment-revisions the verbs revision, entry and difference read (design 47 lines
// 1365-1366); nil serves them not.
revisions assigngraph.Objects
}
// wantsEventSubject is the subject of one of the seat's events.
func wantsEventSubject(event string) string {
return "mesh.seat." + meshwants.Seat + ".event." + event
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
}
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
}
// leased says the controller's lease is handed away.
func (s *wantsTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// values reads every key of a state: from the watched copy when there is one, else key by key.
func (s *wantsTools) values(state string) (map[string][]byte, error) {
if s.watched != nil {
@@ -71,33 +71,44 @@ func (s *wantsTools) values(state string) (map[string][]byte, error) {
return s.direct(state)
}
// direct reads a state whole, in pages where the runtime has them (readAll).
// direct reads a state key by key, for a small state of the seat's own.
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
all, err := readAll(s.st, state)
keys, err := s.st.keys(state)
if err != nil {
return nil, err
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
}
out := make(map[string][]byte, len(all))
for k, v := range all {
out[k] = v.raw
out := map[string][]byte{}
for _, k := range keys {
raw, _, found, err := s.st.get(state, k)
if err != nil {
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
}
if found {
out[k] = raw
}
}
return out, nil
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint: both asked of the indexer's first
// views of modules and manifests (indexerView), never read from its bus state.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]meshcatalog.ModulesByModuleValue, error) {
mods, err := indexerView(s.ask, indexerModules)
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
// under its digest.
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
if s.watched != nil {
return s.watched.Once(manifestsState, digest)
}
raw, _, found, err := s.st.get(manifestsState, digest)
return raw, found, err
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
mods, err := s.values(modulesState)
if err != nil {
return nil, nil, err
}
kept, err := indexerView(s.ask, indexerManifests)
if err != nil {
return nil, nil, err
}
out, held := map[string]catalogue.Manifest{}, map[string]meshcatalog.ModulesByModuleValue{}
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
for name, raw := range mods {
m, err := meshcatalogclient.DecodeModule(raw)
m, err := delivery.DecodeModule(raw)
if err != nil {
continue
}
@@ -105,16 +116,16 @@ func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]mesh
if m.Manifest == "" {
continue
}
mraw, found := kept[m.Manifest]
if !found {
mraw, found, err := s.manifest(m.Manifest)
if err != nil || !found {
continue
}
stored, err := meshcatalogclient.DecodeManifest(mraw)
kept, err := delivery.DecodeManifest(mraw)
if err != nil {
continue
}
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
if man, err := catalogue.Decode([]byte(stored.Manifest)); err == nil {
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
out[name] = man
}
}
@@ -131,7 +142,7 @@ type policyRow struct {
}
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
// back with default; with compare each node's composition and the assigner's findings.
// back with default; with compare list the splits.
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
manifests, _, err := s.manifests()
if err != nil {
@@ -143,27 +154,17 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
chosen, unreadable := assign.DecodePolicies(raw)
if arg(args, "compare") == "true" {
haves, err := s.values(haveState)
composed, err := s.values(composedState)
if err != nil {
return nil, err
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
if splits == nil {
splits = []assign.Split{}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(buckets.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(assign.FindingsState); err != nil {
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
keys := []string{}
@@ -195,14 +196,18 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
// and says policy-changed. The bus is never rolled: a choice for it is refused.
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
if module == "" {
return nil, errors.New("a policy is set for one module: name it")
}
if _, ok := manifests[module]; !ok {
m, ok := manifests[module]
if !ok {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
if catalogue.ProvidesBus(m) {
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
}
if why == "" {
return nil, errors.New("a person's policy says why")
}
@@ -224,8 +229,8 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
}
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, rerr := rec.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
@@ -264,37 +269,37 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
if bus == "" {
return nil, errors.New("the indexer holds no module that provides the bus")
}
newest := held[bus].Fp2
newest := held[bus].FP2
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
h, err := delivery.DecodeHave(raw)
if err != nil || h.Phase == delivery.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
if steps, err := s.ask("seat:"+meshhandacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
out["steps-unread"] = err.Error()
} else {
out["steps"] = steps
}
kept, err := s.values(putsState)
puts, err := s.values(putsState)
if err != nil {
out["puts-unread"] = err.Error()
} else {
mine := map[string]json.RawMessage{}
for k, v := range kept {
if _, module, err := puts.ParseKey(k); err == nil && module == bus {
for k, v := range puts {
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
mine[k] = v
}
}
@@ -303,24 +308,16 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
return out, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>; the tool runner describes them with the
// seat's definition from the seat list (ADR 0635 §1), so the tools carry none.
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
func (s *wantsTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
run := map[string]func(map[string]any) (any, error){
meshwants.VerbUpgrade: func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
meshwants.VerbBus: s.doBus,
}
if s.revisions != nil {
for v, f := range graphVerbs(s.revisions) {
run[v] = f
}
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
"bus": s.doBus,
}
var out []stdio.Tool
for _, v := range meshwants.Verbs() {
if run[v] == nil {
continue // a verb of the seat this module does not serve yet: revision, entry, difference (AS11)
}
out = append(out, stdio.Tool{Name: meshwants.Seat + "." + v, Run: run[v]})
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
}
return out
}
@@ -1,146 +0,0 @@
package main
import (
"context"
"encoding/json"
"sort"
"strings"
"time"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-assigner/internal/assign"
)
// Why a want is what it is (lead-47 2026-11-15, the tools rule: novox's want of the indexer stood unwritten for a day and nothing
// said why). Each pass writes, in the state written beside the pipeline ends, one key `why.<node>.<module>`: the step of the pass
// that decided the want and what it read there. A key whose time stands still says the pass no longer judges that module.
// assigner_why reads them.
// whyPrefix starts every key of a want's why in the state written; a pipeline end's key starts with a module's name.
const whyPrefix = "why."
// whyTool is the tool that reads them.
const whyTool = "assigner_why"
// The steps of the pass that decide a want.
const (
whyNoPipeline = "no-passed-pipeline" // no version of the module released on the branch passed its pipeline: the want written stays
whyPassed = "passed-pipeline" // the want names the newest version released on the branch whose pipeline passed (design 47 H §750)
)
// whyStep is the step that decided a want: a passed pipeline's version, or none yet.
func whyStep(passed bool) string {
if passed {
return whyPassed
}
return whyNoPipeline
}
// wantWhy is one key's value.
type wantWhy struct {
At time.Time `json:"at"`
Branch string `json:"branch,omitempty"`
// Fp, Version and Pipeline are the build the want names and the pipeline whose passed end released it (design 47 H §750).
Fp string `json:"fp,omitempty"`
Version string `json:"version,omitempty"`
Pipeline string `json:"pipeline,omitempty"`
Step string `json:"step"`
// Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before).
Before *mwc.Want `json:"before,omitempty"`
After *mwc.Want `json:"after,omitempty"`
}
func whyKey(node, module string) string { return whyPrefix + node + "." + module }
// sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear.
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome) {
kv := p.kvs[writtenBucket()]
if kv == nil {
return
}
if p.whys == nil {
p.whys = map[string]string{}
}
for _, node := range nodes {
one := w
one.At = time.Now().UTC()
if b, ok := before[node]; ok {
one.Before = &b
}
if out != nil {
if a, ok := out.Wants[node]; ok {
one.After = &a
}
}
same := one
same.At = time.Time{}
cmp, _ := json.Marshal(same)
key := whyKey(node, module)
if p.whys[key] == string(cmp) {
continue
}
raw, err := json.Marshal(one)
if err != nil {
continue
}
if _, err := kv.Put(ctx, key, raw); err != nil {
p.logf("the why of %s on %s is not written: %v", module, node, err)
continue
}
p.whys[key] = string(cmp)
}
}
// whyTools is the tool assigner_why: every want's why, or those of one module or one node.
func whyTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline yet, or a passed pipeline's version), the build, version and pipeline it names, the want before and the one written. A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil),
Run: func(args map[string]any) (any, error) {
ks, err := st.keys(writtenState)
if err != nil {
return nil, err
}
module, node := arg(args, "module"), arg(args, "node")
sort.Strings(ks)
out := map[string]wantWhy{}
for _, k := range ks {
rest, ok := strings.CutPrefix(k, whyPrefix)
if !ok {
continue
}
n, mod, _ := strings.Cut(rest, ".")
if (module != "" && mod != module) || (node != "" && n != node) {
continue
}
raw, _, ok, err := st.get(writtenState, k)
if err != nil || !ok {
continue
}
var w wantWhy
if json.Unmarshal(raw, &w) == nil {
out[rest] = w
}
}
return out, nil
}}}
}
// sayPass writes, under `why.pass`, whether the last pass judged the wants at all, once per change of its words.
func (p *passer) sayPass(ctx context.Context, words string) {
kv := p.kvs[writtenBucket()]
if kv == nil {
return
}
if p.whys == nil {
p.whys = map[string]string{}
}
if p.whys[whyPrefix+"pass"] == words {
return
}
raw, _ := json.Marshal(wantWhy{At: time.Now().UTC(), Step: words})
if _, err := kv.Put(ctx, whyPrefix+"pass", raw); err == nil {
p.whys[whyPrefix+"pass"] = words
}
}
@@ -1,31 +1,45 @@
package main
import (
"context"
"encoding/json"
"errors"
"maps"
"mesh-delivery-assigner/internal/buckets"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/keys"
"github.com/nats-io/nats.go/jetstream"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
)
// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
const noModuleWords = delivery.CandidateNoneNoModule
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
// internal/view/view.go): an assignment stands while it has not ended and has no state.
type assignment struct {
Node string `json:"node"`
Module string `json:"module"`
Build string `json:"build"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
// digest. The watches write it; a pass reads it.
type world struct {
@@ -38,29 +52,11 @@ type world struct {
changes map[string]uint64
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
// ends are the done builds' ends the builder says (build.ended), by module and fingerprint (buildlog.go).
ends builtEnds
// released are the catalog's version-released heard, by module and source revision (buildended.go).
released releasedVersions
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
edges map[string]map[string][]meshcatalog.Edge
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
assignmentKeys *mmac.Keys
// first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the
// branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532),
// and never fixed in code (§7).
first string
// presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389).
presence map[string]presenceSaid
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
running map[string]nodeengine.Running
runningHeard bool
// modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14,
// §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog.
modules map[string]map[string]meshcatalog.ModulesByModuleValue
graphs map[string]map[string]graph.Node
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
trunkBuilt map[string]delivery.BuildLogEntry
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
placement *delivery.PlacementKeys
}
// entryMeta is a value's revision in its bucket and when it was written.
@@ -78,9 +74,7 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(),
running: map[string]nodeengine.Running{},
modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}}
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
}
// putEntry is put, keeping the value's revision and the time it was written.
@@ -93,8 +87,8 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
return
}
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
}
@@ -115,25 +109,42 @@ func (w *world) put(bucket, key string, value []byte) {
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
if value == nil {
delete(w.values[bucket], key)
if bucket == plc.AssignmentsBucket() {
_, _ = w.assignmentKeys.Put(key, nil, 0, true)
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, nil, 0, true)
}
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == plc.AssignmentsBucket() {
if bucket == delivery.PlacementBucket {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.assignmentKeys.Put(key, value, 0, false)
_, _ = w.placement.Put(key, value, 0, false)
}
if bucket == buckets.Have {
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
w.healthy[key] = append(list, h.Fp)
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
w.healthy[key] = append(list, h.FP)
}
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := delivery.HaveKey(node, module)
if err != nil {
return ""
}
w.mu.Lock()
defer w.mu.Unlock()
list := w.healthy[key]
for i := len(list) - 1; i >= 0; i-- {
if list[i] != fp {
return list[i]
}
}
return ""
}
// read is a copy of what a bucket holds now.
func (w *world) read(bucket string) map[string][]byte {
w.mu.Lock()
@@ -145,46 +156,111 @@ func (w *world) read(bucket string) map[string][]byte {
return out
}
// getter reads one key's value; errNotThere when the key does not exist.
type getter func(ctx context.Context, key string) ([]byte, error)
var errNotThere = errors.New("no such key")
// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable.
func (w *world) manifest(digest string) parsed {
// kvGetter reads keys of a bucket.
func kvGetter(kv jetstream.KeyValue) getter {
return func(ctx context.Context, key string) ([]byte, error) {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, errNotThere
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
}
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
w.mu.Lock()
defer w.mu.Unlock()
if p, ok := w.manifests[digest]; ok {
p, ok := w.manifests[digest]
w.mu.Unlock()
if ok {
return p
}
return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"}
}
// assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the
// assignments is the one source of where modules run. The controller's assignments are read no more (issue 774).
func (w *world) assigned() map[string][]string {
placed, _ := w.placed()
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
key, err := delivery.ManifestsKey(digest)
if err != nil {
return parsed{err: err.Error()}
}
return out
value, err := get(ctx, key)
if errors.Is(err, errNotThere) {
return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"}
}
if err != nil {
return parsed{err: err.Error()}
}
m, err := delivery.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
// module's value where it is read (world.mesh).
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
} else {
p = parsed{module: m.Module, manifest: man}
}
w.mu.Lock()
w.manifests[digest] = p
w.mu.Unlock()
return p
}
// placed is each node's modules in the seat's assignments (ADR 0567 §1), and whether any assignments key was read: a key
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
func (w *world) assigned() (map[string][]string, map[string]assignment) {
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
// the lease write's own check refuses that order (ADR 0547 §11).
placed, seeded := w.placed()
byAssignment, all := w.assignedByController()
if away := w.handedAway(); away && seeded {
both, _ := assign.Placed(byAssignment, placed, seeded, away)
return both, map[string]assignment{}
}
both, _ := assign.Placed(byAssignment, placed, seeded, false)
return both, all
}
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
// the controller's lease is handed away, since the placement is then the one source.
func (w *world) placementDiffers() []assign.Pair {
placed, seeded := w.placed()
byAssignment, _ := w.assignedByController()
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
return differ
}
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
func (w *world) handedAway() bool {
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
return away
}
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
for node, p := range w.assignments() {
out[node] = mmac.Assigned(p)
for node, p := range w.placements() {
out[node] = p.Assigned()
}
return out, len(w.read(plc.AssignmentsBucket())) > 0
return out, len(w.read(delivery.PlacementBucket)) > 0
}
// assignments is every node's assignments, assembled from its keys in either form (mmac.Keys).
func (w *world) assignments() map[string]mma.Assignments {
out, _ := w.assignmentKeys.AllRevisions()
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
func (w *world) placements() map[string]delivery.Placement {
out, _ := w.placement.AllRevisions()
return out
}
// placedNodes is every node the seat's assignments names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
@@ -195,187 +271,265 @@ func (w *world) placedNodes() map[string]bool {
return out
}
// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184).
func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue {
w.mu.Lock()
defer w.mu.Unlock()
return maps.Clone(w.modules[branch])
}
// graphOf is the module graph of the catalog half of branch.
func (w *world) graphOf(branch string) graph.Graph {
w.mu.Lock()
defer w.mu.Unlock()
return graph.Graph(maps.Clone(w.graphs[branch]))
}
// branches are the watched branches the catalog half is held of, the trunk's first.
func (w *world) branches() []string {
w.mu.Lock()
defer w.mu.Unlock()
out := slices.Sorted(maps.Keys(w.modules))
if i := slices.Index(out, w.first); i > 0 {
out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first)
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
// name order.
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
out, all := map[string][]string{}, map[string]assignment{}
for _, value := range w.read(provider.AssignmentsBucket) {
var a assignment
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
continue
}
out[a.Node] = append(out[a.Node], a.Module)
all[a.Node+"."+a.Module] = a
}
return out
for _, mods := range out {
sort.Strings(mods)
}
return out, all
}
// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's
// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it.
func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue {
out := map[string]meshcatalog.ModulesByModuleValue{}
for _, b := range w.branches() {
for name, v := range w.trunkOf(b) {
if _, ok := out[name]; !ok {
out[name] = v
}
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
for key, value := range w.read(delivery.ModulesBucket) {
name, err := delivery.ParseModulesKey(key)
if err != nil {
continue
}
if m, err := delivery.DecodeModule(value); err == nil {
out[name] = m
}
}
return out
}
// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the
// value names.
func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) {
w.mu.Lock()
defer w.mu.Unlock()
if w.modules[branch] == nil {
w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{}
// mesh is the mesh as the trunk holds it, for the composition check.
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
m.Assigned, _ = w.assigned()
for name, mod := range w.trunk() {
if mod.Manifest == "" {
continue
}
m.Digests[name] = mod.Manifest
// The indexer's verdict on it, carried in its value (ADR 0578 §3, §5): a refused module is held and named as one
// that cannot be read, and its verdict goes into the composer's input.
if v := mod.VerdictOf(name); v != nil {
if refused, _ := delivery.TakeManifestVerdict(v); refused {
if m.Verdicts == nil {
m.Verdicts = map[string]delivery.ManifestVerdict{}
}
m.Verdicts[name] = *v
m.Unreadable[name] = v.Why()
continue
}
}
p := w.manifest(ctx, manifests, mod.Manifest)
if p.err != "" {
m.Unreadable[name] = p.err
continue
}
m.Manifests[name] = p.manifest
}
w.modules[branch][module], w.graphs[branch][module] = v, n
if m != nil && v.Manifest != "" {
w.manifests[v.Manifest] = *m
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
// assignments alone are composed, as before any placement.
if _, seeded := w.placed(); seeded {
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
if err != nil {
m.InputUnread = err.Error()
} else {
m.Input = &in
}
}
return m
}
// dropModule takes a module out of branch's catalog half.
func (w *world) dropModule(branch, module string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.modules[branch], module)
delete(w.graphs[branch], module)
delete(w.edges[branch], module)
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
// composition it asks resolves, and stands in for what it does not carry).
type worldReader struct {
w *world
manifests getter
}
// putEdges keeps the edges into module's artifacts that branch's catalog entry names.
func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) {
w.mu.Lock()
defer w.mu.Unlock()
if w.edges == nil {
w.edges = map[string]map[string][]meshcatalog.Edge{}
}
if w.edges[branch] == nil {
w.edges[branch] = map[string][]meshcatalog.Edge{}
}
w.edges[branch][module] = edges
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
return r.w.placements(), nil
}
// edgesOf is the edges into module's artifacts in branch's catalog half.
func (w *world) edgesOf(branch, module string) []meshcatalog.Edge {
w.mu.Lock()
defer w.mu.Unlock()
return w.edges[branch][module]
}
// holdsBranch says whether the catalog half of branch has been taken in (even empty).
func (w *world) holdsBranch(branch string) bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.modules[branch] != nil
}
// records is each node's record in its own have key (`<node>.node`): the nodes an assignments proposal is answered by,
// with those assignments names (ADR 0593 §8).
func (w *world) records() map[string]nodeengine.NodeRecord {
out := map[string]nodeengine.NodeRecord{}
for key, value := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
out := map[string]delivery.NodeRecord{}
for key, value := range r.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
return out
return out, nil
}
// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389).
type presenceSaid struct {
away bool
at time.Time
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
return r.w.trunk(), nil
}
// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two
// streams, read whole in either order, leave each machine as its last event says.
func (w *world) heardPresence(machine string, away bool, at time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
if w.presence == nil {
w.presence = map[string]presenceSaid{}
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
if was, ok := w.presence[machine]; ok && at.Before(was.at) {
return
raw, err := r.manifests(ctx, key)
if err != nil {
return nil, err
}
w.presence[machine] = presenceSaid{away: away, at: at}
m, err := delivery.DecodeManifest(raw)
if err != nil {
return nil, err
}
return []byte(m.JSON), nil
}
// awayNow is every machine whose last presence event heard is machine-away.
func (w *world) awayNow() map[string]bool {
w.mu.Lock()
defer w.mu.Unlock()
out := map[string]bool{}
for m, p := range w.presence {
if p.away {
out[m] = true
}
}
return out
}
// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to
// the seat's verb is heard (novox/hq ADR 0668 §3).
func (w *world) seatHeard(seat string) bool {
online := w.online()
for _, value := range w.read(plc.SeatHoldersBucket()) {
var sh mma.SeatHolders
if json.Unmarshal(value, &sh) != nil || sh.Seat != seat {
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
var out map[string]any
prefix := node + "." + module + "."
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
name, ok := strings.CutPrefix(key, prefix)
if !ok || strings.Contains(name, ".") {
continue
}
for _, h := range sh.Holders {
if online[h.Node] {
return true
if v, laid := catalogue.LaidValue(raw); laid {
if out == nil {
out = map[string]any{}
}
out[name] = v
}
}
return out, nil
}
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
return nil, nil
}
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
return nil, nil, nil
}
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
// composes from the manifests the indexer read, and reads no build and no archive.
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
return nil, nil
}
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
}
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
all, err := r.Values(ctx, node, module)
var out map[string]any
for _, k := range keys {
if v, ok := all[k]; ok {
if out == nil {
out = map[string]any{}
}
out[k] = v
}
}
return out, err
}
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
heads := map[string]string{}
for _, p := range c.Pulls {
heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head
}
ch.Held = assign.HeldOf(w.read(keptBucket), heads)
definedIn := map[string]map[string]bool{}
for _, p := range c.Pulls {
if p.Repository == assign.SDKRepository {
ch.SDKHead = p.Head.Head
}
ch.Members[p.Repository] = p.Head.Head
for _, name := range p.Modules {
fp, ok := p.Fps[name]
if !ok || fp.Manifest == "" {
if ok && fp.None2 == noModuleWords {
ch.Removed = append(ch.Removed, name)
continue
}
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
}
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
ch.Rebuilt++
}
got := w.manifest(ctx, manifests, fp.Manifest)
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
return ch, got.err
}
ch.Digests[name] = fp.Manifest
if got.err != "" {
ch.Unreadable[name] = got.err
continue
}
ch.Manifests[got.module] = got.manifest
if definedIn[got.module] == nil {
definedIn[got.module] = map[string]bool{}
}
definedIn[got.module][p.Repository] = true
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
definedIn[got.module][t.Repository] = true
}
}
}
return false
for name, repos := range definedIn {
if len(repos) > 1 {
for r := range repos {
ch.Twice[name] = append(ch.Twice[name], r)
}
sort.Strings(ch.Twice[name])
}
}
return ch, ""
}
// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its
// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2).
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
away := w.awayNow()
out := map[string]bool{}
for _, node := range w.knownNodes() {
out[node] = !away[node]
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
out[node] = p.Online
}
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]mwc.Want {
out := map[string]map[string]mwc.Want{}
for key, value := range w.read(buckets.Wants()) {
node, module, err := mwc.ParseWantKey(key)
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
for key, value := range w.read(delivery.AssignerWantsBucket) {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
continue
}
want, err := mwc.DecodeWant(value)
want, err := delivery.DecodeWant(value)
if err != nil {
continue
}
if out[module] == nil {
out[module] = map[string]mwc.Want{}
out[module] = map[string]delivery.Want{}
}
out[module][node] = want
}
@@ -384,20 +538,24 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
n.Online = w.online()
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if isNode { // what the node's engine reads is the state engine-reads' alone (design 47 H-A-104)
if p, err := delivery.DecodePresence(value); err == nil {
n.Online[node] = p.Online
}
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
continue
}
if h, err := nodeengineclient.DecodeHave(value); err == nil {
if h, err := delivery.DecodeHave(value); err == nil {
if n.Haves[node] == nil {
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
n.Haves[node] = map[string]delivery.Have{}
}
n.Haves[node][module] = h
}
@@ -428,7 +586,7 @@ func (w *world) value(bucket, key string) []byte {
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
// manifest but the controller's reads the controller's grants.
func (w *world) grantsBucket() string { return buckets.Grants }
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
@@ -438,42 +596,76 @@ func (w *world) has(bucket, key string) bool {
return ok
}
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
func (w *world) knownNodes() []string {
set := map[string]bool{}
for node := range w.placedNodes() {
set[node] = true
}
for node := range w.records() {
set[node] = true
// built says whether the builder's build of a module at a fingerprint is done.
func (w *world) built(module, fp string) bool {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return false
}
w.mu.Lock()
for node := range w.presence {
set[node] = true
}
value, ok := w.values[delivery.BuilderBuildsBucket][key]
w.mu.Unlock()
return slices.Sorted(maps.Keys(set))
if !ok {
return false
}
b, err := delivery.DecodeBuilderBuild(value)
return err == nil && b.State == delivery.BuildDone
}
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
if !w.placedNodes()[node] {
return assign.ComposedDigest{}, false
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
//
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
// no want (and the pass says so, unplacedFindings).
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
if w.handedAway() {
if !w.placedNodes()[node] {
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
}
return w.ownComposed(node)
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
return delivery.ComposedDigest{}, false
}
w.mu.Lock()
value, ok := w.values[delivery.ControllerComposedBucket][key]
w.mu.Unlock()
if !ok {
return delivery.ComposedDigest{}, false
}
d, err := delivery.DecodeComposedDigest(value)
return d, err == nil
}
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
if w.handedAway() {
return out // the controller composes no node from the lease write (ADR 0571 §6)
}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
continue
}
if d, err := delivery.DecodeComposedDigest(value); err == nil {
out[node] = d
}
}
return out
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (mdc.Answer, bool) {
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[buckets.Answers][key]
value, ok := w.values[delivery.AnswersBucket][key]
w.mu.Unlock()
if !ok {
return mdc.Answer{}, false
return delivery.Answer{}, false
}
a, err := mdc.DecodeAnswer(value)
a, err := delivery.DecodeAnswer(value)
return a, err == nil
}
@@ -492,32 +684,69 @@ func (w *world) changed(bucket, key string) uint64 {
return w.changes[bucket+"|"+key]
}
// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch).
type logWatch func(handle func(stdio.LogEntry) error) error
// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the
// module's last, becomes the module's running commit. It answers whether the commit changed.
func (w *world) putBuilt(raw []byte) bool {
e, err := delivery.DecodeBuildLogEntry(raw)
if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
have, ok := w.trunkBuilt[e.Module]
if ok && !e.At.After(have.At) {
return false
}
w.trunkBuilt[e.Module] = e
return !ok || have.Commit != e.Commit
}
// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a
// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build.
func (w *world) runningCommits(module string) []string {
w.mu.Lock()
defer w.mu.Unlock()
if e, ok := w.trunkBuilt[module]; ok {
return []string{e.Commit}
}
return nil
}
// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5).
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
key, err := nodeengineclient.HaveNodeKey(node)
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
if err != nil {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
raw, ok := w.read(buckets.Have)[key]
raw, ok := w.read(delivery.HaveBucket)[key]
if !ok {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
h, err := nodeengineclient.DecodeHaveNode(raw)
h, err := delivery.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return assign.ComposedDigest{}, false
return delivery.ComposedDigest{}, false
}
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
Words: "the node composes its own"}, true
}
// unplacedNodes are the nodes whose engine has a have and that no assignments names: they get no
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
// want, since nothing composes them.
func (w *world) unplacedNodes() []string {
if !w.handedAway() {
return nil
}
placed := w.placedNodes()
var out []string
for key := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
for key := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
}
@@ -525,46 +754,3 @@ func (w *world) unplacedNodes() []string {
sort.Strings(out)
return out
}
// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted.
func (w *world) trunkName() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.first
}
// setTrunk says which watched branch is the first.
func (w *world) setTrunk(branch string) {
w.mu.Lock()
defer w.mu.Unlock()
w.first = branch
}
// placedBranches is, by module and node, the watched branch each placement names ("" when it names none).
func (w *world) placedBranches() map[string]map[string]string {
out := map[string]map[string]string{}
for node, a := range w.assignments() {
for _, m := range a.Modules {
if out[m.Module] == nil {
out[m.Module] = map[string]string{}
}
out[m.Module][node] = m.Branch
}
}
return out
}
// assignedOn is the modules the assignments place on node.
func (w *world) assignedOn(node string) []string {
var out []string
for _, m := range w.assignments()[node].Modules {
out = append(out, m.Module)
}
return out
}
// assignmentOf is node's assignments, or false where none is held.
func (w *world) assignmentOf(node string) (mma.Assignments, bool) {
a, ok := w.assignments()[node]
return a, ok
}
@@ -2,17 +2,16 @@ package main
import (
"context"
"mesh-delivery-assigner/internal/buckets"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// fakeManifests is the indexer's manifests bucket in memory.
@@ -20,8 +19,8 @@ type fakeManifests map[string][]byte
func (f fakeManifests) add(t *testing.T, module, body string) string {
t.Helper()
digest := meshcatalogclient.ManifestDigest([]byte(body))
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
digest := delivery.ManifestDigest([]byte(body))
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
if err != nil {
t.Fatal(err)
}
@@ -36,37 +35,127 @@ func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
return nil, errNotThere
}
func fp(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
for m := range modules {
p.Modules = append(p.Modules, m)
}
slices.Sort(p.Modules)
return p
}
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
// another repository as defined twice.
func TestTheChangeOfACandidate(t *testing.T) {
theWorldCountsTheValuesAKeyTakes(t)
f := fakeManifests{}
web := f.add(t, "web", `{"module":"web","version":"2"}`)
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
pull(assign.SDKRepository, 3, nil),
}}
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
if waits != "" {
t.Fatalf("the change waits: %s", waits)
}
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
t.Errorf("change = %+v", ch)
}
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
}
}
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
// saying for what; a manifest that does not decode is the change's, not a wait (one with a field the assigner does not
// know is decoded with it set aside, novox/hq ADR 0578 §6).
func TestAChangeWaitsForItsManifests(t *testing.T) {
f := fakeManifests{}
bad := f.add(t, "web", `{"module":"web","version":"2","requires":"x"}`)
for _, c := range []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
} {
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
if waits == "" {
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
}
}
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
if waits != "" || ch.Unreadable["web"] == "" {
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
}
}
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
// xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"predefined"`) {
t.Fatal("the fixture names no predefined key")
}
f := fakeManifests{}
xterm := f.add(t, "xterm", string(raw))
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
}
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(buckets.Have, "gone"); got != 0 {
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{Accounts: []string{"ace.web"}})
// ADR 0571 clause 9: grants are read from the grants part's bucket in every case, a lease held, handed away, absent or
// unreadable alike, and the controller's own grants count for nothing.
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name string
inPart bool
granted bool
name, lease string
inPart, inController bool
granted bool
}{
{"the part's grant", true, true},
{"no grant", false, false},
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, true},
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, false},
{"a controller holds it, the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, true},
{"a controller holds it, only its own grant", `{"instance":"novox/1"}`, false, true, false},
{"no lease read", ``, true, false, true},
{"an unreadable lease", `[1]`, true, false, true},
} {
p := newPasser(kvSet{}, time.Now())
if row.inPart {
p.w.put(buckets.Grants, gkey, grant)
p := newPasser(buckets{}, time.Now())
if row.lease != "" {
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
}
if got := p.w.grantsBucket(); got != buckets.Grants {
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
}
if row.inController {
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
}
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
@@ -74,41 +163,3 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
}
}
}
// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it.
func putModuleRaw(w *world, branch, module string, raw []byte) {
v, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
panic(err)
}
w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil)
}
// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the
// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main".
func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) {
w := newWorld()
w.setTrunk("develop")
for _, b := range []string{"beta", "develop", "alpha"} {
w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil)
}
if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" {
t.Errorf("branches = %v, want the first watched first, then by name", got)
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(src), `"main"`) {
t.Errorf("%s names the branch main in code (design 47 G §507)", f)
}
}
}
+5 -92
View File
@@ -3,107 +3,20 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.1 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.2 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.16.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk => ../mesh-delivery-sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient => ../mesh-delivery-contracts/go/clients/meshcatalogclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient => ../mesh-delivery-contracts/go/clients/meshdeliveryclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog => ../mesh-delivery-contracts/go/seats/meshcatalog
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery => ../mesh-delivery-contracts/go/seats/meshdelivery
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks => ./asks
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents => ../mesh-delivery-teller/tellerevents
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate => ../mesh-delivery-view/viewstate
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
)
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient => ../mesh-delivery-contracts/go/clients/meshpipelinerunnerclient
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner => ../mesh-delivery-contracts/go/seats/meshpipelinerunner
+28 -8
View File
@@ -1,13 +1,33 @@
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364 h1:UUDPWfUjyerptar+wPwI891jTUAtbbuXori0j3phTE4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3 h1:lb6hz+SBsoCAzsiCCvdtPKyE66Y5Ng/DBAkonAexPzA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
@@ -21,5 +41,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
@@ -1,85 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"regexp"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
func assignerAsk(asking int, a oc.Ask) asks.Ask {
a.Headline, a.Explanation, a.Details, a.OnExpiry = said(a.Headline), said(a.Explanation), said(a.Details), said(a.OnExpiry)
for i := range a.Options {
a.Options[i].Label, a.Options[i].Does = said(a.Options[i].Label), said(a.Options[i].Does)
}
raw, err := json.Marshal(a)
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
}
return asks.Ask{Asking: asking, Ask: raw}
}
// longHex is a digest or fingerprint whole: 64 hexadecimal digits.
var longHex = regexp.MustCompile(`[0-9a-f]{64}`)
// said is words as an ask carries them out of the mesh: each digest or fingerprint cut to its first 12 digits, which the
// router lets pass, where it refuses a long hexadecimal string as a secret's shape (mesh-sdk outward; novox 2026-11-16 01:24:
// every backup ask was refused for its "(composition sha256:…)" and the fingerprints in the node's words, and the operator
// was never asked).
func said(words string) string {
return longHex.ReplaceAllStringFunc(words, func(h string) string { return h[:12] })
}
// AskOf is the router's ask an asks-state value carries.
func AskOf(a asks.Ask) (oc.Ask, error) {
var ask oc.Ask
if err := json.Unmarshal(a.Ask, &ask); err != nil {
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
}
return ask, nil
}
// EncodeAsk is asks.EncodeAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
// the ask's id is meshdeliveryclient.AskingID(key, asking), and it offers at least one option.
func EncodeAsk(key string, a asks.Ask) ([]byte, error) {
if _, err := checkedAsk(key, a); err != nil {
return nil, err
}
return asks.EncodeAsk(key, a)
}
func checkedAsk(key string, a asks.Ask) (oc.Ask, error) {
ask, err := occ.Carried(a.Ask, meshdeliveryclient.AskingID(key, a.Asking))
if err != nil {
return oc.Ask{}, fmt.Errorf("assign: %w", err)
}
return ask, nil
}
// DecodeAsk reads the ask at key (asks.DecodeAsk) with the router's ask it carries.
func DecodeAsk(key string, data []byte) (asks.Ask, oc.Ask, error) {
a, err := asks.DecodeAsk(key, data)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
ask, err := checkedAsk(key, a)
if err != nil {
return asks.Ask{}, oc.Ask{}, err
}
return a, ask, nil
}
// ExpiresAt is when the ask ends unanswered; the zero time for an ask that names none.
func ExpiresAt(a oc.Ask) time.Time {
t, err := occ.Time(a.Expires)
if err != nil {
return time.Time{}
}
return t
}
@@ -2,154 +2,191 @@ package assign
import (
"fmt"
"mesh-delivery-assigner/internal/assignment"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
"lets the wants move to it now.", module, node, short(fp), words, module),
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the version stays held",
About: key,
}}
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
OptionRestore = "restore"
OptionLeaveStopped = "leave-stopped"
)
// The options of the outside ask (design 47 G §706).
const (
OptionGoOn = "go-on"
OptionFailed = "failed"
)
// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
Options: []oc.Option{
{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []asks.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the delivery keeps waiting",
About: about,
})
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module stays as it is on the node",
About: key,
}}
}
// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
Severity: "warning",
Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
// node and module, and cleared (raised false) when the node has a want for it again.
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
key := delivery.AssignerModule + ".neither." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
}
headline := fmt.Sprintf("%s on %s is on no list", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
if h.Bus {
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
asks.NeedsYou, needs, module, node, h.Words)}
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) asks.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
c.Resolved = LiftedWords(module, fp) + "."
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = mwc.BackupAgain
OptionBackupSkip = mwc.BackupSkip
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
)
// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
const (
OptionRevertHold = "hold"
OptionRevertLive = mwc.RevertLive
)
// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
const KindRevertHeld = "revert-held"
// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
// the asking, 0 the first, as the backup ask's.
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
module, node, short(fp), kept, form, words, module),
Options: []oc.Option{
{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
OnExpiry: "the module keeps running its newer build",
About: key,
})
}
// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
func RevertAnswerWords(option string, asking int) string {
return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
}
// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
// operator's one message is the ask.
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
module, reason, kept, form, module),
Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
}
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
// asked again, for the rewritten key carried the id the router already kept).
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + need.Words,
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
Options: []oc.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
Options: []asks.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the node keeps what it runs",
About: key,
})
}}
}
// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
@@ -157,6 +194,18 @@ func NextBackupAsking(after int) int {
return max(after+1, 2)
}
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a delivery.AssignerAsk) string {
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
return m[1]
}
return ""
}
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
@@ -164,36 +213,36 @@ func BackupAnswerWords(option string, asking int) string {
}
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: meshdeliveryclient.AskingID(key, asking),
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
module, short(fp), node, words, module),
Options: []oc.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Options: []asks.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
},
Who: occ.Operator,
Expires: occ.FormatTime(now.Add(AskLasts)),
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module keeps running what it has",
About: key,
})
}}
}
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
needs := "answer whether to back up again or go on without a backup"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}
@@ -1,259 +1,148 @@
package assign
import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/puts"
"regexp"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// 1297/B9 (the assigner's half): the backup ask is a valid ask of the router and of mesh-sdk's layout at its key, of a kind the
// view may answer; the hold and restore asks went with the hold of a fingerprint (design 47 H §750, H-A-55).
func TestTheAssignersAsks(t *testing.T) {
t.Run("backup", theBackupAskIsValid)
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
}
// Issue 1966, note 151: an ask the teller says ended at the operator channel without an answer (its event ask-ended) is
// written again under a new asking, an id of its own and a new expiry, one row per kind: the rules' asks are; the
// authorising ask of a proposal (ADR 0567 §5, §11) and a put's record ask (ADR 0546) are not, nor an asking the key no longer
// holds, an end asking again would not change, or a body that does not read.
func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
now := asked.Add(8 * 24 * time.Hour)
digest := "sha256:" + strings.Repeat("e", 64)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
assignmentsKey, _ := asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")
need := RecordNeed{Node: "novox", Digest: digest, Judgement: JudgedMovesData}
record := RecordAsk(recordKey, "novox", "nats", fNew, need, 0, asked) // written before #195: no asking
put := puts.Put{FP: fNew, Why: "w", At: asked, Step: puts.Person, Ask: recordKey}
byPut := RecordAsk(recordKey, "novox", "nats", fNew, need, PutAsking(put), asked)
other := puts.Put{FP: fNew, Why: "w", At: asked.Add(time.Hour), Step: puts.Person, Ask: recordKey}
assignments := assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(assignmentsKey, AskingSeconds(asked)), Headline: "h",
Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(occ.ApproveLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: OptionYes, Label: "Yes", Does: "d", Level: occ.Approve}}})
for _, c := range []struct {
name string
key string
a *asks.Ask // the ask the key holds; nil for none
ended string // the id of the asking that ended; "" for the one the key holds
outcome string
put *puts.Put
why string // "" when it is written again
}{
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeCancelled},
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
{name: "record", key: recordKey, a: &record, outcome: occ.OutcomeCancelled},
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
{name: "another asking stands", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", AskingSeconds(now), now)),
ended: meshdeliveryclient.AskingID(backupKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
{name: "taken away", key: backupKey, ended: "backup-x", outcome: occ.OutcomeExpired, why: "taken away"},
{name: "refused", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
{name: "replaced", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
} {
t.Run(c.name, func(t *testing.T) {
var current []byte
ended := c.ended
if c.a != nil {
var err error
if current, err = EncodeAsk(c.key, *c.a); err != nil {
t.Fatal(err)
}
if ended == "" {
ended = mustAsk(t, *c.a).ID
}
}
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: c.key, Ask: ended, Outcome: c.outcome})
if err != nil {
t.Fatal(err)
}
e, err := tellerevents.DecodeEnded(body)
if err != nil {
t.Fatal(err)
}
value, why := AskedAgain(e, current, c.put, now)
if c.why != "" {
if value != nil || !strings.Contains(why, c.why) {
t.Fatalf("AskedAgain = %s, %q; want nothing written, %q", value, why, c.why)
}
return
}
if why != "" {
t.Fatalf("AskedAgain says %q, want the ask written again", why)
}
a, ask, err := DecodeAsk(c.key, value)
if err != nil {
t.Fatal(err)
}
if a.Asking <= c.a.Asking || ask.ID != meshdeliveryclient.AskingID(c.key, a.Asking) || ask.ID == ended {
t.Errorf("asking %d under %s, want a new asking above %d under an id of its own, not %s", a.Asking, ask.ID, c.a.Asking, ended)
}
if err := occ.Check(ask, now); err != nil || !ExpiresAt(ask).Equal(now.Add(AskLasts).Truncate(time.Second)) {
t.Errorf("the router refuses the ask written again (%v), or it lapses at %s, want %s", err, ExpiresAt(ask), now.Add(AskLasts))
}
})
}
for _, raw := range []string{`{"v":1,"key":"k","ask":"a"}`, `{"v":1,"key":"k","ask":"a","outcome":"expired","more":1}`} {
if _, err := tellerevents.DecodeEnded([]byte(raw)); err == nil {
t.Errorf("DecodeEnded(%s) takes a body that is not an ended ask", raw)
}
}
}
// Issue 1966 note 155, ADR 0668 §4: an ended ask the teller said before the assigner's consumer existed is not delivered to
// it, so at bind the assigner reads the operator-channel seat's asks: the query reaches back to the earliest held asking,
// and of the answer only the held askings that lapsed or were cancelled are taken, as the teller's event says them; an
// answered or another asker's ask, and one the key no longer holds, are not.
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
holdKey, _ := asks.BackupAskKey("g14", "nats", fNew)
restoreKey, _ := asks.BackupAskKey("ace", "nats", fNew)
hold := BackupAsk(holdKey, "g14", "nats", fNew, "w", AskingSeconds(asked), asked)
restore := BackupAsk(restoreKey, "ace", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
backup := BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked.Add(-time.Hour)) // no asking
held := map[string][]byte{}
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
v, err := EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
}
held[key] = v
}
if _, ok := ChannelAsksQuery(nil); ok {
t.Error("with no ask held the channel is asked")
}
q, ok := ChannelAsksQuery(held)
if !ok || q.State != "ended" || q.Bound != ChannelAsksBound || q.Since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) {
t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339))
}
at := func(key string, a asks.Ask, state string) string {
return fmt.Sprintf(`{"key":"t.%s","state":%q}`, mustAsk(t, a).ID, state)
}
answer := `{"asks":[` + strings.Join([]string{at(holdKey, hold, occ.OutcomeExpired), at(restoreKey, restore, occ.OutcomeChosen),
at(backupKey, backup, occ.OutcomeCancelled), `{"key":"t.hold-gone","state":"expired"}`, `{"key":"open","state":"open"}`}, ",") + `],"cut":"c"}`
ended, cut, err := EndedAtChannel(held, []byte(answer))
want := []tellerevents.Ended{{Key: backupKey, Ask: mustAsk(t, backup).ID, Outcome: occ.OutcomeCancelled},
{Key: holdKey, Ask: mustAsk(t, hold).ID, Outcome: occ.OutcomeExpired}}
slices.SortFunc(want, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
if err != nil || cut != "c" || !slices.Equal(ended, want) {
t.Errorf("EndedAtChannel = %+v, %q, %v; want %+v and the answer's cut", ended, cut, err, want)
}
if _, _, err := EndedAtChannel(held, []byte(`[1]`)); err == nil {
t.Error("an answer that does not read is taken")
}
}
func ptr(a asks.Ask) *asks.Ask { return &a }
func theBackupAskIsValid(t *testing.T) {
key, err := asks.BackupAskKey("ace", "mail", fNew)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
// its key, of a kind the view may answer.
func TestTheHoldAskIsValid(t *testing.T) {
key, err := delivery.HoldAskKey("mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := BackupAsk(key, "ace", "mail", fNew, "health failed, phase running: connection refused", 0, now)
if err := occ.Check(mustAsk(t, a), now); err != nil {
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
// The ask names the machine whose backup failed and the backup's words.
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
// 1906/H3: the ask names the machine whose check failed and the hold's words.
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
}
// An ask written again is another asking, under an id of its own.
again := BackupAsk(key, "ace", "mail", fNew, "x", 7, now)
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
t.Errorf("the asking again = %d %s", again.Asking, mustAsk(t, again).ID)
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
}
if _, err := EncodeAsk(key, again); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
good := BackupAsk(key, "ace", "mail", fNew, "x", 0, time.Unix(100, 0))
other := good
other.Asking = 3 // its id is the asking 0's
empty := good
ask := mustAsk(t, good)
ask.Options = nil
empty.Ask = assignerAsk(0, ask).Ask
for name, a := range map[string]asks.Ask{"another id": other, "no option": empty, "no ask": {}} {
if _, err := EncodeAsk(key, a); err == nil {
t.Errorf("%s: written", name)
}
}
})
}
// 1952/WM6: the assigner's conditions are findings.
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func TestTheRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range a.Ask.Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
}
}
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
func TestTheAssignersConditions(t *testing.T) {
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings)
}
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
t.Errorf("condition = %+v", c)
}
}
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
// findings bucket cannot be written to at all.
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear; it is said straight to the router while the take-over
// key follows the controller, and only as a finding from the take-over.
func saidConditionsAreFindings(t *testing.T) {
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
for _, c := range []struct {
raised, cleared oc.Condition
raised, cleared asks.Condition
key string
}{
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]mcseat.Finding{}, []string{}, 0
f := Findings{Put: func(f mcseat.Finding) error {
puts[views.FindingKey(f.Scope, f.ID, f.Kind)] = f
return nil
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
say := f.Say(func(oc.Condition) error { said++; return nil })
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
keeper := false
f := Findings{Put: func(k string, v []byte) error {
got, err := health.DecodeFinding(v)
puts[k] = got
return err
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }}
say := f.Say(func(asks.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
t.Fatalf("%s: %v", c.key, err)
}
got, ok := puts[c.key]
if !ok || got.Severity != c.raised.Severity || got.Headline != c.raised.Headline || got.Observer != assignment.Module() ||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
got.Subject.Machine != c.raised.Subject.Machine || said != 1 {
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
}
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
}
if err := (Findings{}).Say(func(oc.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
keeper = true
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 {
t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err)
}
}
if _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
t.Error("a condition not the assigner's is a finding")
}
}
@@ -262,132 +151,103 @@ func saidConditionsAreFindings(t *testing.T) {
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := asks.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, 2, now)
bkey, err := asks.BackupAskKey("ace", "mail", fNew)
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
b := BackupAsk(bkey, "ace", "mail", fNew, "no space left", 0, now)
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
v := RevertAsk(vkey, "ace", "mail", fNew, "records", "3", "restore point missing: none of form 3", 0, now)
for key, a := range map[string]asks.Ask{rkey: r, bkey: b, vkey: v} {
if err := occ.Check(mustAsk(t, a), now); err != nil {
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
if err := a.Ask.Check(now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := EncodeAsk(key, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := viewstate.ViewAnswerKey(key); err != nil {
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", mustAsk(t, r))
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", r.Ask)
}
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
t.Errorf("the backup ask = %+v", mustAsk(t, b))
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
t.Errorf("the backup ask = %+v", b.Ask)
}
// the id binds the ask to its number (review round 1, item 4); its key binds it to the build (design 47 H §750: a want names no composition)
if again := BackupAsk(bkey, "ace", "mail", fNew, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
}
// novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole.
held := BackupAsk(bkey, "novox", "mail", fNew, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) {
t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation)
if BackupAskDigest(b) != dgst {
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
}
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
t.Errorf("two answers of the same option are one want value")
}
// design 47 G §710: the revert ask names the build, the kept data, its form and the node's words, and offers hold and live;
// its finding is delivery.<node>.<module>.revert-held, a warning that needs nothing, the ask being the one message.
if va := mustAsk(t, v); va.Headline != "The revert of mail on ace has no restore point" || va.Options[0].ID != OptionRevertHold ||
va.Options[1].ID != mwc.RevertLive || len(va.Options) != 2 || va.OnExpiry != "the module keeps running its newer build" ||
!strings.Contains(va.Explanation, "records") || !strings.Contains(va.Explanation, "form 3") || !strings.Contains(va.Explanation, short(fNew)) ||
!strings.Contains(va.Explanation, "Started on the live data, the older build reads data written in the newer form.") {
t.Errorf("the revert ask = %+v", va)
}
if f := RevertHeldFinding("ace", "mail", "records", "3", "restore point missing").Finding(now); views.FindingKey(f.Scope, f.ID, f.Kind) != "delivery.ace.mail.revert-held" ||
f.Severity != "warning" || f.Needs != "Nothing for you to do." ||
f.Summary != "The revert of mail on ace is held: restore point missing (records, form 3); the ask about it waits for your answer" {
t.Errorf("the revert finding = %+v", f)
}
if RevertAnswerWords(OptionRevertLive, 1) != `the operator answered "live" to the revert ask (asking 2)` {
t.Errorf("the words of a live answer: %q", RevertAnswerWords(OptionRevertLive, 1))
}
c := BackupCondition("ace", "mail", fNew, "no space left", true)
if _, err := occ.EncodeCondition(c); err != nil {
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
dgst := "sha256:" + strings.Repeat("b", 64)
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
t.Fatalf("bus %v: %v", bus, err)
}
if err := occ.Check(mustAsk(t, a), now); err != nil {
if err := a.Ask.Check(now); err != nil {
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
}
if _, err := EncodeAsk(rkey, a); err != nil {
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := viewstate.ViewAnswerFits(viewstate.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
yes := mustAsk(t, a).Options[0]
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
// acknowledge), as it did through the sdk before.
if viewed == nil {
if o, ok := occ.OptionOf(mustAsk(t, a), OptionYes); ok && o.Level != "" && o.Level != occ.Acknowledge {
viewed = fmt.Errorf("only a channel that proves who answered takes the option %s", o.ID)
}
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
yes := a.Ask.Options[0]
switch {
case !strings.Contains(mustAsk(t, a).Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, mustAsk(t, a).Explanation)
case bus && (yes.Level != occ.Approve || yes.Binds == "" || viewed == nil || expiresAt(t, a).After(now.Add(occ.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, expiresAt(t, a))
case !bus && (yes.Level != occ.Acknowledge || viewed != nil):
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
}
later := put
later.At = now.Add(time.Minute)
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
if mustAsk(t, b).ID == mustAsk(t, a).ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, mustAsk(t, b).ID, mustAsk(t, a).ID)
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dgst)); err != nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
}
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
t.Errorf("bus: the yes performs another composition's act")
}
}
}
// mustAsk is the router's ask an asks-state value carries.
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
t.Helper()
ask, err := AskOf(a)
if err != nil {
t.Fatal(err)
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
t.Errorf("condition = %+v", c)
}
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
return ask
}
func expiresAt(t *testing.T, a asks.Ask) time.Time {
t.Helper()
return ExpiresAt(mustAsk(t, a))
}
@@ -1,23 +0,0 @@
package assign
import (
"mesh-delivery-assigner/internal/assignment"
"strings"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// SeatWaitsCondition is the assigner's word on the seat mesh-module-assigner while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) oc.Condition {
key := assignment.Module() + ".seat-states"
subject := oc.ConditionSubject{Scope: "module", ID: assignment.Module()}
if len(unbound) == 0 {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, Resolved: "the seat mesh-module-assigner holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: occ.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -0,0 +1,368 @@
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
// and writes nothing itself: the program hands it what it watched and writes what it answers.
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
const SDKRepository = "novox/mesh-sdk"
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
"node-engine's validator would refuse"
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
// why, and each node's open assignments.
type Mesh struct {
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Assigned map[string][]string
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
Input *compose.Input
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
// nodes and their assignments alone, and says so in its warnings.
InputUnread string
// Verdicts are the indexer's refusals of trunk manifests, by module, as their values carry them (novox/hq ADR 0578
// §3): each such module is in Unreadable too, and the composer's input carries the verdict (§5).
Verdicts map[string]delivery.ManifestVerdict
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
// member (empty when none), and how many modules it rebuilds.
type Change struct {
Group, Heads string
Manifests map[string]catalogue.Manifest
Unreadable map[string]string
Removed []string
Twice map[string][]string
SDKHead string
Rebuilt int
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
// Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR
// 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named.
Held []delivery.KeptValue
}
// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another
// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key.
func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue {
var out []delivery.KeptValue
for _, raw := range kept {
var k delivery.KeptValue
if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" {
continue
}
if head, ok := heads[k.Pull]; ok && head == k.Head {
out = append(out, k)
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i], out[j]
return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key
})
return out
}
// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers
// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the
// declaration refuses, or a key it does not declare, is a failure.
func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) {
for _, k := range held {
var v any
if err := json.Unmarshal(k.Value, &v); err != nil {
said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err))
continue
}
d, declared := in.Catalogue[k.Module].Settings[k.Key]
if !declared {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull))
continue
}
if problems := catalogue.ValueProblems(d, v); len(problems) > 0 {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module,
k.Node, k.Pull, strings.Join(problems, "; ")))
continue
}
at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node })
if at < 0 {
said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node))
continue
}
if in.Nodes[at].Settings == nil {
in.Nodes[at].Settings = map[string][]catalogue.Layer{}
}
in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}})
said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved))
}
return said, fails
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
// mesh-placement judges it.
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
moved[name] = true
}
for name := range c.Unreadable {
moved[name] = true
}
for _, name := range c.Removed {
moved[name] = true
}
provided, required := map[string]bool{}, map[string]bool{}
for name := range moved {
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
for _, o := range man.Provides {
provided[o.Name] = true
}
}
for _, r := range c.Manifests[name].Requires {
required[r] = true
}
}
var out []string
for node, modules := range m.Assigned {
for _, mod := range modules {
man := m.Manifests[mod]
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
in := compose.Input{Catalogue: catalog}
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
for _, mod := range assigned[n] {
for _, cl := range catalog[mod].Claims {
scope := cl.Scope
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
scope = seat.Scope
}
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
}
}
}
return in
}
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
var fails []string
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Unreadable) {
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
machines := Touched(m, c)
v.Machines = machines
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
}
trunk := input(m.Manifests, m.Assigned)
changed := map[string]catalogue.Manifest{}
for name, man := range m.Manifests {
changed[name] = man
}
for name, man := range c.Manifests {
changed[name] = man
}
for _, name := range c.Removed {
delete(changed, name)
}
after := input(changed, m.Assigned)
laid, heldFails := layHeld(&after, c.Held)
v.Warnings = append(v.Warnings, laid...)
for _, f := range heldFails {
failAt("", f)
}
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
switch {
case now == nil:
case before != nil:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
}
}
if c.SDKHead != "" {
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
}
return finish(v, m, c, machines, fails)
}
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
if c.Rebuilt > delivery.CompositionRebuildWarning {
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
}
v.Warnings = append(v.Warnings, NotJudged)
if could := CouldRun(m, c, machines); len(could) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
}
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
}
if len(machines) == 0 {
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
}
if len(fails) > 0 {
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
return v
}
v.State = delivery.CompositionPass
return v
}
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
// the facts snapshot), so each is named as not judged.
func CouldRun(m Mesh, c Change, composed []string) []string {
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for name := range c.Manifests {
if !slices.Contains(modules, name) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
func grantsOf(man catalogue.Manifest) string {
var offers []string
for _, o := range man.Provides {
offers = append(offers, o.Name)
}
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
}
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
func GrantsAlone(m Mesh, c Change, composed []string) []string {
provisions := map[string]bool{}
altered := false
for name, man := range c.Manifests {
before, had := m.Manifests[name]
if had && grantsOf(before) == grantsOf(man) {
continue
}
altered = true
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
provisions[r] = true
}
}
if !altered {
return nil
}
var out []string
for node, modules := range m.Assigned {
if slices.Contains(composed, node) {
continue
}
for _, mod := range modules {
man := m.Manifests[mod]
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
out = append(out, node)
break
}
}
}
sort.Strings(out)
return out
}
// withOverlay is base with every manifest of over laid over it, a copy.
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
out := make(map[string]catalogue.Manifest, len(base)+len(over))
for name, man := range base {
out[name] = man
}
for name, man := range over {
out[name] = man
}
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -0,0 +1,169 @@
package assign
import (
"encoding/json"
"slices"
"strconv"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func meshOf() Mesh {
return Mesh{
Manifests: map[string]catalogue.Manifest{
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
"web": {Module: "web", Version: "1"},
},
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
}
}
func hasWarning(v delivery.CompositionVerdict, part string) bool {
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
}
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
// the machine and the mesh-sdk it composed with, and says what it did not judge.
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
v := Compose(meshOf(), c, "v0.1.14-x", 0)
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
}
if !hasWarning(v, "issue 1461") {
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
// 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged
// against the head's declaration; one of another pull request or another head is not laid.
web, err := catalogue.Decode([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`))
if err != nil {
t.Fatal(err)
}
kept := map[string][]byte{}
for i, k := range []delivery.KeptValue{
{Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"},
{Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)},
{Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)},
} {
kept[strconv.Itoa(i)], _ = json.Marshal(k)
}
held := HeldOf(kept, map[string]string{"novox/x#1": "h1"})
if len(held) != 1 || string(held[0].Value) != "81" {
t.Fatalf("held = %+v, want #1's at its head alone", held)
}
c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held}
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") {
t.Errorf("composition = %+v", v)
}
c.Held[0].Value = json.RawMessage(`"eighty"`)
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") {
t.Errorf("a held value the declaration refuses: %+v", v)
}
}
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
}
}
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
// consumer's machine is touched when its provider changes.
func TestAMachineTheChangeBreaksFails(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
}
}
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
m := meshOf()
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
v := Compose(m, c, "sdk", 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
}
}
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
func TestUnreadableTwiceAndWide(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
v := Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
t.Fatalf("composition = %+v", v)
}
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
v = Compose(meshOf(), c, "sdk", 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
t.Fatalf("composition = %+v", v)
}
}
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
v := Compose(meshOf(), c, "v0.1.14-own", 0)
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
}
c.SDKHead = ""
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
}
}
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
// assigned nowhere touches none, and the verdict says so.
func TestTouched(t *testing.T) {
m := meshOf()
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
t.Errorf("touched by store = %v", got)
}
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
t.Errorf("touched by store with a consumer on g14 = %v", got)
}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
t.Errorf("an added module assigned nowhere: %+v", v)
}
}
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
// names none of the second.
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
m := meshOf()
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
m.Assigned["novox"] = []string{"bus"}
m.Assigned["shanks"] = []string{"db"}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
v := Compose(m, c, "sdk", 0)
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
}
if !hasWarning(v, "whose grants alone the change alters: novox") {
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
}
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
}
}
@@ -1,45 +0,0 @@
package assign
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
// judgement); neither and unchanged are the assigner's own.
type Judgement = string
// The judgements of a move.
const (
// JudgedRecreates is a move that recreates a container.
JudgedRecreates = nec.JudgedRecreates
// JudgedMovesData is a move that gives the module's data another place.
JudgedMovesData = nec.JudgedMovesData
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
JudgedNeither = "neither"
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
JudgedCannot = nec.JudgedCannot
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
// changes nothing on the node, so it needs no person's yes.
JudgedUnchanged = "unchanged"
)
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
// the module). Words say why, and are required for a move that cannot be judged.
//
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
type ComposedDigest struct {
V int `json:"v"`
Digest string `json:"digest"`
Judgement Judgement `json:"judgement"`
Against string `json:"against,omitempty"`
Words string `json:"words,omitempty"`
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
// record hold (ADR 0539 clause 7).
UserListOnly bool `json:"user-list-only,omitempty"`
Policy string `json:"policy,omitempty"`
PolicyFrom string `json:"policy-from,omitempty"`
PolicyWhy string `json:"policy-why,omitempty"`
}
@@ -0,0 +1,263 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
const (
CaseTrunk = "trunk"
CaseChange = "change"
)
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
const SDKDir = "go"
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
// them, the node-engine applies them. Each is judged at its repository.
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
for name, digest := range m.Digests {
trunk[name] = composer.Manifest{Digest: digest}
change[name] = composer.Manifest{Digest: digest}
}
for name, digest := range c.Digests {
change[name] = composer.Manifest{Digest: digest}
}
for _, name := range c.Removed {
delete(change, name)
}
in, fromClaims := plainInput(m.Assigned), true
if m.Input != nil {
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
in = *m.Input
in.Catalogue = nil
fromClaims = len(in.Holdings) == 0
}
// Each trunk manifest the indexer refused goes in with its verdict, so the composer holds and names it (ADR 0578 §5);
// a manifest the change brings has no verdict of the indexer yet, and the checker's judge judges it (§2).
changeVerdicts := map[string]delivery.ManifestVerdict{}
for name, v := range m.Verdicts {
if _, changed := c.Digests[name]; !changed {
changeVerdicts[name] = v
}
}
changeCase := composer.Case{Name: CaseChange, Manifests: change, Verdicts: changeVerdicts}
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk, Verdicts: m.Verdicts}, changeCase}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
changed := map[string]composer.Manifest{}
for name, digest := range c.Digests {
changed[name] = composer.Manifest{Digest: digest}
}
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
}
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
raw, err := json.Marshal(req)
if err != nil {
return delivery.ComposeJob{}, err
}
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
Package: composer.Package, Request: raw, Reader: reader}, nil
}
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
// composer derives the seats held from the claims, as input does).
func plainInput(assigned map[string][]string) compose.Input {
var in compose.Input
nodes := make([]string, 0, len(assigned))
for n := range assigned {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
}
return in
}
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
type Ran struct {
State string
Reason string
Verdict *composer.Verdict
}
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
// failure, named.
func RanOf(step delivery.Step) Ran {
r := Ran{State: step.State, Reason: step.Reason}
if step.State != delivery.StepPassed {
return r
}
var a delivery.ComposeAnswer
var v composer.Verdict
if err := json.Unmarshal(step.Result, &a); err != nil {
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
}
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
}
r.Verdict = &v
return r
}
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
if v == nil {
return composer.CaseVerdict{}, false
}
for _, c := range v.Cases {
if c.Name == name {
return c, true
}
}
return composer.CaseVerdict{}, false
}
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
// the group holds one in its repository, else a build its node runs.
type Reader struct {
Module, Commit string
Member bool
Ran Ran
}
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk cannot decode fails too
// (the group would deliver a reader that cannot read it). A reader that cannot read a changed manifest is a warning,
// never a failure: it decodes leniently and sets a field it does not know aside (novox/hq ADR 0578 §6, which changes
// ADR 0482 §5).
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
machines := Touched(m, c)
v.Machines = machines
var fails []string
if head.State != delivery.StepPassed {
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
v.State = delivery.CompositionFail
return v
}
if head.Verdict.SDK != "" {
v.SDK = head.Verdict.SDK
}
trunk, okT := caseOf(head.Verdict, CaseTrunk)
change, okC := caseOf(head.Verdict, CaseChange)
if !okT || !okC {
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
return v
}
failAt := func(machine, words string) {
if v.Machine == "" && machine != "" {
v.Machine = machine
}
fails = append(fails, words)
}
for _, name := range sortedKeys(c.Digests) {
if r := change.Manifests[name]; !r.Accepted {
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
}
}
for _, name := range sortedKeys(trunk.Manifests) {
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
continue
}
if _, before := m.Unreadable[name]; before {
continue // already unreadable on the trunk: not the group's doing
}
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
}
for _, name := range sortedKeys(c.Twice) {
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
}
for _, node := range machines {
for _, name := range c.Removed {
if slices.Contains(m.Assigned[node], name) {
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
}
}
before, now := trunk.Machines[node], change.Machines[node]
switch {
case now.Composes:
case !before.Composes:
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
default:
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
}
}
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
if m.InputUnread != "" {
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
"cannot be read from the bus ("+m.InputUnread+")")
}
return finish(v, m, c, machines, fails)
}
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
// from before issue 1621): Go's words for a package its module does not hold.
func NoComposer(reason string) bool {
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
if strings.Contains(reason, words) {
return true
}
}
return false
}
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
for _, r := range readers {
at := r.Module + " at " + r.Commit
if r.Member {
at = r.Module + "'s member head " + r.Commit
}
if r.Ran.State != delivery.StepPassed {
if NoComposer(r.Ran.Reason) {
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
"composer command (before issue 1621)", at))
continue
}
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
continue
}
read, ok := caseOf(r.Ran.Verdict, CaseChange)
if !ok {
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
continue
}
for _, name := range sortedKeys(c.Digests) {
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
continue
}
// ADR 0578 §6: no longer a failure; the reader sets what it does not know aside, and the part that needs it takes
// the newer package (ADR 0573 §2).
warnings = append(warnings, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; it reads the rest "+
"once it takes mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, c.SDKHead))
}
}
return fails, warnings
}
@@ -0,0 +1,225 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
const sdkHead = "1111111111111111111111111111111111111111"
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
var raws = map[string]string{
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
"d-web": `{"module": "web", "version": "1"}`,
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
}
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
func digestedMesh() Mesh {
m := meshOf()
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
return m
}
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
func webShares(members map[string]string) Change {
// As an older reader refused it; the field set aside by the lenient decoder is named the same (ADR 0578 §6).
m, _ := catalogue.Decode([]byte(raws["d-web2"]))
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
Unreadable: map[string]string{"web": m.SetAside()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
}
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
t.Helper()
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if err := composer.Fill(&req, func(d string) ([]byte, error) {
if r, ok := read[d]; ok {
return []byte(r), nil
}
return []byte(raws[d]), nil
}); err != nil {
t.Fatal(err)
}
v, err := composer.Run(req)
if err != nil {
t.Fatal(err)
}
v.SDK = "(devel) " + job.Fetch["commit"].(string)
vr, _ := json.Marshal(v)
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
}
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
// change by digest, never the manifests' bytes, composing the machines the change touches.
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
c := webShares(nil)
job, err := HeadJob(digestedMesh(), c)
if err != nil {
t.Fatal(err)
}
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
t.Fatalf("job = %+v", job)
}
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
At: testTime, Compose: &job}
if _, err := delivery.EncodeStepAsk(ask); err != nil {
t.Fatalf("the ask is refused by its layout: %v", err)
}
req, err := composer.Decode(job.Request)
if err != nil {
t.Fatal(err)
}
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
t.Fatalf("cases = %+v", req.Cases)
}
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
}
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
m := digestedMesh()
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
placedJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
placed, err := composer.Decode(placedJob.Request)
if err != nil {
t.Fatal(err)
}
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
v := FromComposer(digestedMesh(), c, head, nil, 0)
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
t.Fatalf("with the head's composer: %+v", v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
}
}
// 1623/AC1, AC2 as ADR 0578 §6 changes them: the group of mesh-sdk and mesh-catalog adding and using a field, without a
// controller member, passes with a warning naming the field and mesh-controller, whose running build pins an older
// mesh-sdk that cannot read it: a reader sets a field it does not know aside, so it fails nothing. With a controller
// member pinned to the SDK head it passes with no such warning.
func TestAFieldTheRunningControllerCannotReadIsAWarning(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
running := "3333333333333333333333333333333333333333"
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
if err != nil {
t.Fatal(err)
}
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
t.Fatalf("reader job = %+v", rjob)
}
// The running controller's mesh-sdk, older than the lenient decoder, refuses web2 for its field.
old := Ran{State: delivery.StepPassed, Verdict: &composer.Verdict{V: composer.Version, SDK: "v0.1.13", Cases: []composer.CaseVerdict{
{Name: CaseChange, Manifests: map[string]composer.ManifestRead{"web": {Refused: `json: unknown field "shares"`}}}}}}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "shares") || !hasWarning(v, "mesh-controller at "+running) {
t.Fatalf("without a controller member: %+v, want a pass warning of the field and mesh-controller", v)
}
member := "2222222222222222222222222222222222222222"
pinned := run(t, rjob, asTheHeadReads)
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
if v.State != delivery.CompositionPass {
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
}
}
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
// failed for another cause fails the group, loud.
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
t.Fatalf("a reader with no composer: %+v", v)
}
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
t.Fatalf("a reader whose composer failed: %+v", v)
}
}
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
c := webShares(nil)
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
t.Fatalf("a head composer that did not run: %+v", v)
}
job, _ := HeadJob(digestedMesh(), c)
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "requires": "x"}`}
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
}
// ADR 0578 §5: a trunk manifest the indexer refused goes into the composer's input with its verdict, in both cases;
// the composer holds and names it, and the group is not blamed for it.
m := digestedMesh()
m.Verdicts = map[string]delivery.ManifestVerdict{"mail": {Module: "mail", State: delivery.VerdictRefused, Problems: []string{"it is wrong"}}}
m.Unreadable = map[string]string{"mail": m.Verdicts["mail"].Why()}
job, _ = HeadJob(m, c)
var req composer.Request
if err := json.Unmarshal(job.Request, &req); err != nil || !req.Cases[0].Verdicts["mail"].Refused() || !req.Cases[1].Verdicts["mail"].Refused() {
t.Fatalf("the request carries no verdict: %+v %v", req.Cases, err)
}
head := run(t, job, asTheHeadReads)
if trunk, _ := caseOf(head.Verdict, CaseTrunk); !strings.Contains(trunk.Manifests["mail"].Refused, "refused the manifest of mail: it is wrong") {
t.Fatalf("the composer did not name the refusal: %+v", trunk.Manifests["mail"])
}
if v = FromComposer(m, c, head, nil, 0); v.State != delivery.CompositionPass {
t.Fatalf("a refusal on the trunk blamed the group: %+v", v)
}
}
@@ -1,139 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"mesh-delivery-assigner/internal/puts"
)
// AskLasts is how long an ask of the assigner's rules stands at the operator channel before it lapses unanswered: the hold,
// restore, record, backup, outside and revert asks.
const AskLasts = 7 * 24 * time.Hour
// writtenAgainOnEnd is, per kind of the assigner's asks, whether an asking that ended at the channel without an answer while
// its key stands is written again under a new asking (issue 1966, note 151), or why not. A rule's ask waits for a person
// as long as its key stands, so it is asked again; the authorising ask of a proposal ends the proposal when it lapses
// (ADR 0567 §5) and is asked again by the seat's verb `again` (§11).
var writtenAgainOnEnd = map[string]string{
asks.AskKindHold: "",
asks.AskKindRestore: "",
asks.AskKindBackup: "",
asks.AskKindOutside: "", // raised again at once while the wait stands (design 47 G §706); the wait's end takes the key away
asks.AskKindRecord: "",
asks.AskKindRevert: "", // raised again at once while the hold stands (design 47 G §710); one answered hold is not asked again
asks.AskKindAssignments: "a proposal's authorising ask that lapsed ends the proposal (ADR 0567 §5); the verb again asks it again (§11)",
}
// AskedAgain is the ask at key written again after the teller said its asking ended at the channel without an answer (the
// teller's event ask-ended): the value to write by compare-and-set over current, or why nothing is written. Nothing is written
// for an outcome other than a lapse or a cancellation, for a key that holds another asking now (an answer to it, or a newer
// asking, came since), for a kind its own rule ends (writtenAgainOnEnd), and for a record ask a put asked (put, nil for none:
// the put at the record ask's node and module), since its put is refused when it lapses (ADR 0546). The new asking is
// AskingSeconds(now), above the old one, under an id of its own, standing AskLasts from now.
func AskedAgain(e tellerevents.Ended, current []byte, put *puts.Put, now time.Time) ([]byte, string) {
if e.Outcome != occ.OutcomeExpired && e.Outcome != occ.OutcomeCancelled {
return nil, fmt.Sprintf("it ended %s, which asking again would not change", e.Outcome)
}
k, err := asks.ParseAnyAskKey(e.Key)
if err != nil {
return nil, err.Error()
}
why, known := writtenAgainOnEnd[k.Kind]
if !known {
return nil, "the assigner asks no " + k.Kind + " ask"
}
if why != "" {
return nil, why
}
if current == nil {
return nil, "the key is taken away: nothing is asked"
}
a, ask, err := DecodeAsk(e.Key, current)
if err != nil {
return nil, err.Error()
}
if ask.ID != e.Ask {
return nil, "the key holds the asking " + ask.ID + " now, not the one that ended"
}
if k.Kind == asks.AskKindRecord && put != nil && put.Ask == e.Key && PutAsking(*put) == a.Asking {
return nil, "a put asked it, and its put is refused when it lapses (ADR 0546)"
}
asking := max(AskingSeconds(now), a.Asking+1)
ask.ID, ask.Expires = meshdeliveryclient.AskingID(e.Key, asking), occ.FormatTime(now.Add(AskLasts))
value, err := EncodeAsk(e.Key, assignerAsk(asking, ask))
if err != nil {
return nil, err.Error()
}
return value, ""
}
// ChannelAsksBound is the most bytes of asks the assigner asks the operator-channel seat's verb asks for at its consumer's
// bind: every ended asking of its held asks fits.
const ChannelAsksBound = 1 << 20
// ChannelAsksQuery is the arguments of the operator-channel seat's verb asks that answer, at the assigner's consumer's bind,
// every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks
// opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false
// when no held ask reads, and then nothing is asked.
func ChannelAsksQuery(held map[string][]byte) (query oc.AsksInput, ok bool) {
var since time.Time
for key, value := range held {
a, ask, err := DecodeAsk(key, value)
if err != nil {
continue
}
opened := ExpiresAt(ask).Add(-AskLasts)
if a.Asking >= 2 {
if at := time.Unix(int64(a.Asking), 0); at.Before(opened) {
opened = at
}
}
if !ok || opened.Before(since) {
since, ok = opened, true
}
}
if !ok {
return oc.AsksInput{}, false
}
return oc.AsksInput{State: "ended", Since: since.Add(-time.Minute).UTC().Format(time.RFC3339), Bound: ChannelAsksBound}, true
}
// EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there
// expired or cancelled, as the teller says it in its event ask-ended, so the assigner writes it again by the rules of that
// event (AskedAgain), with its compare-and-set: an ended ask said before the assigner's consumer existed is not delivered
// to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read
// is the error.
func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) {
var got oc.AsksAnswer // the seat's declared answer: each ask's key and its state, and the cut
if err := json.Unmarshal(answer, &got); err != nil {
return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err)
}
byID := map[string]string{}
for key, value := range held {
if _, ask, err := DecodeAsk(key, value); err == nil {
byID[ask.ID] = key
}
}
for _, v := range got.Asks {
if v.State != occ.OutcomeExpired && v.State != occ.OutcomeCancelled {
continue
}
_, id, ok := oc.ParseAsksByAskerIDKey(v.Key)
if key, held := byID[id]; ok && held {
ended = append(ended, tellerevents.Ended{Key: key, Ask: id, Outcome: v.State})
}
}
slices.SortFunc(ended, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
return ended, got.Cut, nil
}

Some files were not shown because too many files have changed in this diff Show More