Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bca305d198 | ||
|
|
4bff5f408f | ||
|
|
0df4fd9480 | ||
|
|
cb17d59745 |
@@ -2,9 +2,3 @@ vendor/
|
||||
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
|
||||
modules/*/cmd/*/mesh-*
|
||||
!modules/*/cmd/*/mesh-*.go
|
||||
# `go build .` at a module's root leaves its program there too (the view's, met a second time).
|
||||
modules/*/mesh-*
|
||||
# The delivery SDK's commands (design 47 G §464), built in their own directory or at the SDK's root.
|
||||
/modules/mesh-delivery-sdk/cmd/seatgen/seatgen
|
||||
/modules/mesh-delivery-sdk/seatgen/seatgen
|
||||
/modules/mesh-delivery-sdk/mesh-pack
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
|
||||
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
|
||||
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
|
||||
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
|
||||
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
|
||||
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
|
||||
@@ -1,293 +0,0 @@
|
||||
// Package asks holds the layout of the assigner's state `asks`: the keys of the asks the assigner puts to a person and the
|
||||
// value it keeps at each, which the assigner writes and the view and the teller read (novox/hq ADR 0634 §1, ADR 0635 §5). It
|
||||
// is the assigner's own published package. The gate's merge ask (`merge.<group>.<heads hash>`) needs proof of who answered
|
||||
// (ADR 0421 §1), so it is answered on the phone alone until issue 953 closes. The assigner's asks need no such proof, and the
|
||||
// view may answer them: the hold of a fingerprint whose health check failed (`hold.<module>.<fingerprint hex>`: release it or
|
||||
// keep it held), the restore of a data module set back (`restore.<node>.<module>.<fingerprint hex>`: restore and lose newer
|
||||
// writes, or leave it stopped, ADR 0429 §6), a backup that failed before an apply, a move a person must allow for a module of
|
||||
// policy record, and the authorising ask of a proposal of assignments.
|
||||
package asks
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/kept"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
const (
|
||||
// AskKindHold is the kind of the assigner's ask about a held fingerprint.
|
||||
AskKindHold = "hold"
|
||||
// AskKindRestore is the kind of the assigner's ask about a data module set back.
|
||||
AskKindRestore = "restore"
|
||||
// AskKindBackup is the kind of the assigner's ask about a backup before an apply that failed: back up again, or go on
|
||||
// without one (novox/hq ADR 0539 clause 16).
|
||||
AskKindBackup = "backup"
|
||||
// AskKindRecord is the kind of the assigner's ask about a move a person must allow, for a module of policy record: a
|
||||
// composition judged to recreate a container or move data, or one that cannot be judged (ADR 0539 clause 14).
|
||||
AskKindRecord = "record"
|
||||
// AskKindAssignments is the kind of the authorising ask about a proposal of assignments (novox/hq ADR 0567 §5):
|
||||
// `assignments.<proposal id>`. The kind needs no proof, as the bus's put does not: its yes is an Approve, which only a
|
||||
// channel proving who answered takes and the view never does (ViewAnswerFits).
|
||||
AskKindAssignments = "assignments"
|
||||
// AskKindOutside is the kind of the assigner's ask about a moved module whose failing health check has waited past its bound
|
||||
// on an outside cause (design 47 G §706): go on, or count it as failed. `outside.<node>.<module>.<fingerprint hex>`.
|
||||
AskKindOutside = "outside"
|
||||
// AskKindRevert is the kind of the assigner's ask about a held revert, one with no restore point (design 47 G §710): keep it
|
||||
// held, or start the reverted build on the live data. `revert.<node>.<module>.<fingerprint hex>`, the reverted build's.
|
||||
AskKindRevert = "revert"
|
||||
)
|
||||
|
||||
// AskKey is an ask's key read whole: its Kind, and the parts its kind has: a merge's Group and heads Hash, a hold's Module
|
||||
// and fingerprint FP, a restore's, a backup's, an outside's and a revert's Node, Module and FP, a record's Node, Module and the digest Composed of the
|
||||
// composition it would allow, the assignments' Proposal.
|
||||
type AskKey struct {
|
||||
Proposal string
|
||||
Kind string
|
||||
Group string
|
||||
Hash string
|
||||
Node string
|
||||
Module string
|
||||
FP string
|
||||
Composed string
|
||||
}
|
||||
|
||||
// AskNeedsProof says whether an ask of this kind must prove who answered it, so the view may not answer it: the merge's
|
||||
// Approve does (ADR 0421 §1, issue 953), the assigner's hold and restore do not. An unknown kind is refused.
|
||||
func AskNeedsProof(kind string) (bool, error) {
|
||||
switch kind {
|
||||
case meshdeliveryclient.AskKindMerge:
|
||||
return true, nil
|
||||
case AskKindHold, AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert:
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("asks: %q is not a kind of ask, %s, %s, %s, %s, %s, %s, %s or %s", kind, meshdeliveryclient.AskKindMerge, AskKindHold,
|
||||
AskKindRestore, AskKindBackup, AskKindRecord, AskKindAssignments, AskKindOutside, AskKindRevert)
|
||||
}
|
||||
|
||||
// HoldAskKey is the key of the ask about a module's held fingerprint: `hold.<module>.<hex>`.
|
||||
func HoldAskKey(module, fingerprint string) (string, error) {
|
||||
key, err := builds.BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindHold + "." + key, nil
|
||||
}
|
||||
|
||||
// RestoreAskKey is the key of the ask about a data module set back on a node from a fingerprint:
|
||||
// `restore.<node>.<module>.<hex>`.
|
||||
func RestoreAskKey(node, module, fingerprint string) (string, error) {
|
||||
if err := keys.CheckName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
key, err := builds.BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindRestore + "." + node + "." + key, nil
|
||||
}
|
||||
|
||||
// OutsideAskKey is the key of the ask about a module's failing check on a node that waited past its bound on an outside cause:
|
||||
// `outside.<node>.<module>.<hex>`.
|
||||
func OutsideAskKey(node, module, fingerprint string) (string, error) {
|
||||
key, err := RestoreAskKey(node, module, fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindOutside + strings.TrimPrefix(key, AskKindRestore), nil
|
||||
}
|
||||
|
||||
// OutsideAbout is what the outside ask is about: `<node>.<module>.<hex>` (design 47 G §706).
|
||||
func OutsideAbout(key string) string { return strings.TrimPrefix(key, AskKindOutside+".") }
|
||||
|
||||
// BackupAskKey is the key of the ask about a module's failed backup on a node before it applies a fingerprint:
|
||||
// `backup.<node>.<module>.<hex>`.
|
||||
func BackupAskKey(node, module, fingerprint string) (string, error) {
|
||||
part, err := catalogkey.KeyPart(fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key, err := nodeengineclient.ComposedKey(node, module, part)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindBackup + "." + key, nil
|
||||
}
|
||||
|
||||
// RevertAskKey is the key of the ask about a held revert of a module on a node to the reverted build's fingerprint:
|
||||
// `revert.<node>.<module>.<hex>` (design 47 G §710).
|
||||
func RevertAskKey(node, module, fingerprint string) (string, error) {
|
||||
key, err := BackupAskKey(node, module, fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindRevert + strings.TrimPrefix(key, AskKindBackup), nil
|
||||
}
|
||||
|
||||
// AssignmentsAskKey is the key of the authorising ask about a proposal of assignments: `assignments.<proposal id>` (ADR 0567 §5).
|
||||
func AssignmentsAskKey(proposal string) (string, error) {
|
||||
if err := meshmoduleassignerclient.CheckProposalID(proposal); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindAssignments + "." + proposal, nil
|
||||
}
|
||||
|
||||
// RecordAskKey is the key of the ask about a move of a module on a node to the composition whose digest it names:
|
||||
// `record.<node>.<module>.<digest hex>`, so a new revision at the same fingerprint is asked again (ADR 0539 clause 14).
|
||||
func RecordAskKey(node, module, composed string) (string, error) {
|
||||
have, err := nodeengineclient.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return AskKindRecord + "." + have + "." + strings.TrimPrefix(composed, "sha256:"), nil
|
||||
}
|
||||
|
||||
// ParseAnyAskKey reads an ask key of any kind (a merge, hold, restore, backup, record, assignments, outside or revert ask); it refuses an unknown
|
||||
// kind and a key whose parts do not fit its kind, naming the key.
|
||||
func ParseAnyAskKey(key string) (AskKey, error) {
|
||||
kind, rest, _ := strings.Cut(key, ".")
|
||||
refused := func(err error) (AskKey, error) {
|
||||
return AskKey{}, fmt.Errorf("asks: %q is not an ask key: %w", key, err)
|
||||
}
|
||||
switch kind {
|
||||
case meshdeliveryclient.AskKindMerge:
|
||||
_, group, hash, err := meshdeliveryclient.ParseAskKey(key)
|
||||
if err != nil {
|
||||
return AskKey{}, err
|
||||
}
|
||||
return AskKey{Kind: kind, Group: group, Hash: hash}, nil
|
||||
case AskKindHold:
|
||||
module, fp, err := builds.ParseBuilderBuildKey(rest)
|
||||
if err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
return AskKey{Kind: kind, Module: module, FP: fp}, nil
|
||||
case AskKindRestore:
|
||||
node, build, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return refused(fmt.Errorf("a restore ask is restore.<node>.<module>.<fingerprint hex>"))
|
||||
}
|
||||
if err := keys.CheckName("node", node); err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
module, fp, err := builds.ParseBuilderBuildKey(build)
|
||||
if err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
|
||||
case AskKindOutside:
|
||||
k, err := ParseAnyAskKey(AskKindRestore + "." + rest)
|
||||
if err != nil {
|
||||
return refused(fmt.Errorf("an outside ask is outside.<node>.<module>.<fingerprint hex>"))
|
||||
}
|
||||
k.Kind = kind
|
||||
return k, nil
|
||||
case AskKindBackup, AskKindRevert:
|
||||
node, module, part, isNode, err := nodeengineclient.ParseComposedKey(rest)
|
||||
if err != nil || isNode {
|
||||
return refused(fmt.Errorf("a %s ask is %s.<node>.<module>.<fingerprint hex>", kind, kind))
|
||||
}
|
||||
fp, err := catalogkey.ParseKeyPart(part)
|
||||
if err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
return AskKey{Kind: kind, Node: node, Module: module, FP: fp}, nil
|
||||
case AskKindAssignments:
|
||||
if err := meshmoduleassignerclient.CheckProposalID(rest); err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
return AskKey{Kind: kind, Proposal: rest}, nil
|
||||
case AskKindRecord:
|
||||
parts := strings.Split(rest, ".")
|
||||
if len(parts) != 3 {
|
||||
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
|
||||
}
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(parts[0] + "." + parts[1])
|
||||
if err != nil || isNode {
|
||||
return refused(fmt.Errorf("a record ask is record.<node>.<module>.<digest hex>"))
|
||||
}
|
||||
composed := "sha256:" + parts[2]
|
||||
if err := kept.CheckDigest(composed, "a record ask's composition digest"); err != nil {
|
||||
return refused(err)
|
||||
}
|
||||
return AskKey{Kind: kind, Node: node, Module: module, Composed: composed}, nil
|
||||
}
|
||||
_, err := AskNeedsProof(kind)
|
||||
return refused(err)
|
||||
}
|
||||
|
||||
// Ask is the value of a key in the assigner's state asks: the router's ask whole, as the assigner composes it, so the teller
|
||||
// publishes it unchanged. Asking counts the times the assigner wrote the ask at this key, as the gate's ask does: 0 or 1 for
|
||||
// the first, n for the n-th, and the ask's id is meshdeliveryclient.AskingID(key, Asking), so each asking has its own id and
|
||||
// an answer to an earlier asking is never taken for a later one. The ask is the operator-channel seat's as its JSON: its id,
|
||||
// its options and its digest are checked by the assigner and the teller through operator-channel's client (novox/hq ADR
|
||||
// 0627 §4).
|
||||
type Ask struct {
|
||||
V int `json:"v"`
|
||||
Ask json.RawMessage `json:"ask"`
|
||||
Asking int `json:"asking,omitempty"`
|
||||
}
|
||||
|
||||
func (a Ask) check(key string) error {
|
||||
ask, err := ParseAnyAskKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if proof, _ := AskNeedsProof(ask.Kind); proof {
|
||||
return fmt.Errorf("asks: the assigner asks no %s ask: it needs proof of who answered", ask.Kind)
|
||||
}
|
||||
if a.Asking < 0 {
|
||||
return fmt.Errorf("asks: the ask's asking %d is negative", a.Asking)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if len(bytes.TrimSpace(a.Ask)) == 0 || json.Unmarshal(a.Ask, &fields) != nil || fields == nil {
|
||||
return fmt.Errorf("asks: the value carries no router's ask")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeAsk is a, the ask at key, as JSON at keys.Version; it refuses a key of a kind that needs proof or of no kind, a
|
||||
// negative asking, and a value carrying no router's ask.
|
||||
func EncodeAsk(key string, a Ask) ([]byte, error) {
|
||||
if err := a.check(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.V = keys.Version
|
||||
return json.Marshal(a)
|
||||
}
|
||||
|
||||
// DecodeAsk reads the ask at key; it refuses an unknown version (keys.ErrUnknownVersion) and what EncodeAsk refuses.
|
||||
func DecodeAsk(key string, data []byte) (Ask, error) {
|
||||
var a Ask
|
||||
if err := kept.Decode(data, &a); err != nil {
|
||||
return Ask{}, err
|
||||
}
|
||||
if err := a.check(key); err != nil {
|
||||
return Ask{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// SameAsk says whether two asks are the same value: the same version, asking and router's ask, its JSON compared compacted.
|
||||
func SameAsk(a, b Ask) bool {
|
||||
return a.V == b.V && a.Asking == b.Asking && bytes.Equal(compacted(a.Ask), compacted(b.Ask))
|
||||
}
|
||||
|
||||
func compacted(raw json.RawMessage) []byte {
|
||||
var b bytes.Buffer
|
||||
if json.Compact(&b, raw) != nil {
|
||||
return raw
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The keys equal what the assigner has always written (the expected strings are the trunk's own output for the same input),
|
||||
// and each reads back to its parts.
|
||||
func TestTheAssignersAskKeysKeepTheirFormsAndTheValueItsRules(t *testing.T) {
|
||||
fp := "src2:" + strings.Repeat("ab", 32)
|
||||
hex := strings.Repeat("ab", 32)
|
||||
digest := "sha256:" + strings.Repeat("cd", 32)
|
||||
hold, _ := HoldAskKey("gitea", fp)
|
||||
restore, _ := RestoreAskKey("n1", "gitea", fp)
|
||||
backup, _ := BackupAskKey("n1", "novox.be", fp)
|
||||
assignments, _ := AssignmentsAskKey("p-20261028t183000-1a2b3c")
|
||||
record, _ := RecordAskKey("n1", "gitea", digest)
|
||||
revert, _ := RevertAskKey("n1", "nats", fp)
|
||||
for key, want := range map[string]struct {
|
||||
got string
|
||||
back AskKey
|
||||
}{
|
||||
"hold": {hold, AskKey{Kind: "hold", Module: "gitea", FP: fp}},
|
||||
"restore": {restore, AskKey{Kind: "restore", Node: "n1", Module: "gitea", FP: fp}},
|
||||
"backup": {backup, AskKey{Kind: "backup", Node: "n1", Module: "novox.be", FP: fp}},
|
||||
"assignments": {assignments, AskKey{Kind: "assignments", Proposal: "p-20261028t183000-1a2b3c"}},
|
||||
"record": {record, AskKey{Kind: "record", Node: "n1", Module: "gitea", Composed: digest}},
|
||||
"revert": {revert, AskKey{Kind: "revert", Node: "n1", Module: "nats", FP: fp}},
|
||||
} {
|
||||
back, err := ParseAnyAskKey(want.got)
|
||||
if err != nil || back != want.back {
|
||||
t.Errorf("%s: ParseAnyAskKey(%q) = %+v, %v; want %+v", key, want.got, back, err, want.back)
|
||||
}
|
||||
}
|
||||
for got, want := range map[string]string{hold: "hold.gitea." + hex, restore: "restore.n1.gitea." + hex, backup: "backup.n1.novox=2ebe." + hex,
|
||||
assignments: "assignments.p-20261028t183000-1a2b3c", record: "record.n1.gitea." + strings.Repeat("cd", 32), revert: "revert.n1.nats." + hex} {
|
||||
if got != want {
|
||||
t.Errorf("key %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
merge, err := ParseAnyAskKey("merge.fix/1=2e2.d106c7950d28")
|
||||
if err != nil || merge.Kind != "merge" || merge.Group != "fix/1.2" || merge.Hash != "d106c7950d28" {
|
||||
t.Errorf("a merge ask key: %+v, %v", merge, err)
|
||||
}
|
||||
for _, bad := range []string{"", "other.x", "hold.gitea", "hold.gitea.src2:" + hex, "restore.n1", "backup.n1.node", "backup.n1.gitea." + hex[:60],
|
||||
"assignment.AB", "record.n1.gitea." + hex[:10], "record.n1.node." + hex, "merge.g", "revert.n1.node", "revert.n1.nats." + hex[:60]} {
|
||||
if _, err := ParseAnyAskKey(bad); err == nil {
|
||||
t.Errorf("ParseAnyAskKey(%q) reads a wrong key", bad)
|
||||
}
|
||||
}
|
||||
if proof, err := AskNeedsProof("merge"); err != nil || !proof {
|
||||
t.Error("a merge ask needs proof")
|
||||
}
|
||||
for _, kind := range []string{"hold", "revert"} {
|
||||
if proof, err := AskNeedsProof(kind); err != nil || proof {
|
||||
t.Errorf("a %s ask needs no proof", kind)
|
||||
}
|
||||
}
|
||||
if _, err := AskNeedsProof("view"); err == nil {
|
||||
t.Error("an unknown kind of ask is taken")
|
||||
}
|
||||
|
||||
// The value: the router's ask whole; an ask of a kind that needs proof, a negative asking and no ask are refused.
|
||||
good := Ask{Ask: []byte(`{"id":"x"}`), Asking: 2}
|
||||
data, err := EncodeAsk(hold, good)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeAsk: %v", err)
|
||||
}
|
||||
back, err := DecodeAsk(hold, data)
|
||||
if err != nil || !SameAsk(back, Ask{V: keys.Version, Ask: []byte(` {"id": "x"} `), Asking: 2}) {
|
||||
t.Errorf("DecodeAsk = %+v, %v", back, err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
key string
|
||||
a Ask
|
||||
}{"a merge ask": {"merge.g.d106c7950d28", good}, "a negative asking": {hold, Ask{Ask: good.Ask, Asking: -1}}, "no ask": {hold, Ask{}},
|
||||
"a null ask": {hold, Ask{Ask: []byte("null")}}, "an unknown kind": {"other.x", good}} {
|
||||
if _, err := EncodeAsk(c.key, c.a); err == nil {
|
||||
t.Errorf("EncodeAsk takes %s", name)
|
||||
}
|
||||
}
|
||||
if _, err := DecodeAsk(hold, []byte(`{"v":2,"ask":{}}`)); !errors.Is(err, keys.ErrUnknownVersion) {
|
||||
t.Errorf("DecodeAsk of another version = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
)
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// askAgainTool asks one of the assigner's asks again by hand (issue 1966, ADR 0668 §5): the recovery of an ask that ended at
|
||||
// the operator channel without an answer when the event that would have asked it again was not heard.
|
||||
const askAgainTool = "assigner_ask_again"
|
||||
|
||||
// asksState and answersState are the assigner's asks and the teller's answers, as the runtime names them.
|
||||
const (
|
||||
asksState = "asks"
|
||||
answersState = buckets.TellerModule + ".answers"
|
||||
)
|
||||
|
||||
// askAgainByHand writes the ask at key again under a new asking, by the rules the teller's ask-ended writes it by (assign.AskedAgain:
|
||||
// a proposal's authorising ask and a put's record ask are not), so the teller asks it again; an asking the operator answered
|
||||
// is refused, naming the answer. The act is recorded through the seat mesh-hand-acts.
|
||||
func askAgainByHand(st store, ask func(string, any) (json.RawMessage, error), key, why, by string, now time.Time) (any, error) {
|
||||
switch {
|
||||
case key == "":
|
||||
return nil, errors.New("name the ask's key, as the tool asks lists it")
|
||||
case why == "":
|
||||
return nil, errors.New("a hand ask again says why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("a hand ask again names who asks it, and the runtime proved no caller")
|
||||
}
|
||||
current, _, found, err := st.get(asksState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the assigner holds no ask under %s: the tool asks lists them", key)
|
||||
}
|
||||
_, routerAsk, err := assign.DecodeAsk(key, current)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw, _, found, err := st.get(answersState, key); err == nil && found {
|
||||
if a, err := meshdeliveryclient.DecodeAnswer(raw); err == nil && strings.HasPrefix(a.Warrant, routerAsk.ID+"@") {
|
||||
return nil, fmt.Errorf("the asking %s was answered %q (%s): it is not asked again", routerAsk.ID, a.Option, a.Words)
|
||||
}
|
||||
}
|
||||
var put *puts.Put
|
||||
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRecord {
|
||||
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
|
||||
if raw, _, found, err := st.get(putsState, pkey); err == nil && found {
|
||||
if p, err := puts.Decode(raw); err == nil {
|
||||
put = &p
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
value, refused := assign.AskedAgain(tellerevents.Ended{Key: key, Ask: routerAsk.ID, Outcome: occ.OutcomeCancelled}, current, put, now)
|
||||
if refused != "" {
|
||||
return nil, errors.New(refused)
|
||||
}
|
||||
if err := st.put(asksState, key, value); err != nil {
|
||||
return nil, fmt.Errorf("the ask %s is not written again: %w", key, err)
|
||||
}
|
||||
_, next, _ := assign.DecodeAsk(key, value)
|
||||
taken, rerr := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askAgainTool + " " + key, Why: why + " (" + by + ")", Cause: "ask-again"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
recorded = "not recorded: " + rerr.Error()
|
||||
}
|
||||
return map[string]any{"key": key, "ended": routerAsk.ID, "asked": next.ID, "by": by, "hand-act": recorded}, nil
|
||||
}
|
||||
|
||||
// askAgainTools is the tool over the runtime's states.
|
||||
func askAgainTools(st store, ask func(string, any) (json.RawMessage, error), now func() time.Time) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: askAgainTool, Description: "Ask one of the assigner's asks of the operator again (issue 1966): the ask " +
|
||||
"under key is written again under a new asking, which the teller asks on the operator channel, as when the channel says it " +
|
||||
"ended without an answer. Refused for an asking the operator answered, a proposal's authorising ask (the seat's verb again " +
|
||||
"asks it) and a record ask a put asked. Recorded as a hand act.",
|
||||
Input: schema(map[string]string{"key": "the ask's key, as the tool asks lists it", "why": "why it is asked again"},
|
||||
[]string{"key", "why"}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
key, _ := a["key"].(string)
|
||||
why, _ := a["why"].(string)
|
||||
return askAgainByHand(st, ask, key, why, stdio.CallerOf(a), now())
|
||||
}}}
|
||||
}
|
||||
@@ -1,239 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// askEndedSubject is the teller's event ask-ended, as the manifest's consumes names it (`mesh-delivery-teller.ask-ended`).
|
||||
var askEndedSubject = process.Account{Module: buckets.TellerModule}.EventSubject(tellerevents.AskEnded)
|
||||
|
||||
// staleOf says why the ask at a key is stale, as the pass reads it (passer.staleAsk), or "" when it is not or when the
|
||||
// trunk is not read; nil reads nothing.
|
||||
type staleOf func(key string) string
|
||||
|
||||
// staleOfWorld is staleOf over the world the pass reads: the same rule as the pass, over the same cache.
|
||||
func staleOfWorld(w *world) staleOf {
|
||||
p := &passer{w: w}
|
||||
return func(key string) string {
|
||||
trunk := w.trunk()
|
||||
if len(trunk) == 0 {
|
||||
return ""
|
||||
}
|
||||
return p.staleAsk(key, trunk)
|
||||
}
|
||||
}
|
||||
|
||||
// hearAskEnded hears the teller's ask-ended on the assigner's one hub of its durable consumer on EVENTS (registered before
|
||||
// the hub starts): an ask of the assigner's that ended at the operator channel without an answer while its key stands is
|
||||
// written again under a new asking, so the teller asks it again (issue 1966, note 151). asksKV is the assigner's asks; the
|
||||
// puts are opened at each event, since a put's record ask is not written again (ADR 0546).
|
||||
func hearAskEnded(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, logf func(string, ...any)) error {
|
||||
return hub.Handle(askEndedSubject, func(m jetstream.Msg) {
|
||||
var putsKV jetstream.KeyValue
|
||||
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
|
||||
putsKV = kv
|
||||
}
|
||||
if err := askAgain(ctx, asksKV, putsKV, stale, m.Data(), time.Now(), logf); err != nil {
|
||||
logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// channelAsksVerb is the operator-channel seat's verb asks, read at the assigner's consumer's bind (a seat verb: no module
|
||||
// named).
|
||||
const channelAsksVerb = "seat:" + oc.Seat + "." + oc.VerbAsks
|
||||
|
||||
// readEndedAtBind registers on the hub, before it starts, one read of the operator-channel seat's asks at each bind of the
|
||||
// assigner's consumer: an ask-ended the teller said before the consumer existed is not delivered to it (issue 1966 note 155,
|
||||
// ADR 0668 §4), so each held ask whose asking ended there expired or cancelled is written again as that event writes it
|
||||
// (askAgain, with every rule and its compare-and-set). The read is a kept call (CallKept), made off the hub's start.
|
||||
func readEndedAtBind(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, logf func(string, ...any)) {
|
||||
if asksKV == nil || bus == nil {
|
||||
return
|
||||
}
|
||||
hub.OnBound(func(created time.Time) {
|
||||
go func() {
|
||||
if err := endedAtBind(ctx, js, asksKV, stale, bus, wait, created, time.Now(), logf); err != nil {
|
||||
logf("the operator channel's ended asks are not read at the bind of %s (%v): an ask that ended there before is asked again when the teller says it ended", created.Format(time.RFC3339), err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// callWait is how a kept call the assigner waits on is said (novox/hq ADR 0668 §3): heard says the callee's machine is
|
||||
// heard, now is the clock, and findings the assigner's findings, where the call unanswered past
|
||||
// meshconditionsclient.CallUnansweredAfter is the finding call-unanswered while heard, cleared when it is answered.
|
||||
type callWait struct {
|
||||
heard func() bool
|
||||
now func() time.Time
|
||||
findings assign.Findings
|
||||
}
|
||||
|
||||
// asksCallID is the finding's id for the bind's read of the operator channel's asks.
|
||||
const asksCallID = "operator-channel-asks"
|
||||
|
||||
// keep makes call again after every transport fault (busretry: a request that reached no one, or one whose deadline
|
||||
// passed) until it is answered or ctx ends, each try itself a kept call; past the bound with the callee heard it writes
|
||||
// the finding call-unanswered, and clears it when answered. No timer: each try is the wait.
|
||||
func (cw callWait) keep(ctx context.Context, id, callee string, call func() (json.RawMessage, error), logf func(string, ...any)) (json.RawMessage, error) {
|
||||
clock := cw.now
|
||||
if clock == nil {
|
||||
clock = time.Now
|
||||
}
|
||||
since, said := clock(), false
|
||||
key := mcc.Key(mcc.ScopeCall, id, mcc.KindCallUnanswered)
|
||||
for {
|
||||
answer, err := call()
|
||||
if err == nil || !(busretry.Undelivered(err) || busretry.TimedOut(err)) || ctx.Err() != nil {
|
||||
if said && cw.findings.Delete != nil {
|
||||
if derr := cw.findings.Delete(key); derr != nil {
|
||||
logf("the finding %s is not cleared: %v", key, derr)
|
||||
}
|
||||
}
|
||||
return answer, err
|
||||
}
|
||||
if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil {
|
||||
continue
|
||||
}
|
||||
eerr := cw.findings.Put(assign.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since).Finding(clock()))
|
||||
if eerr != nil {
|
||||
logf("the finding %s is not written: %v", key, eerr)
|
||||
continue
|
||||
}
|
||||
said = true
|
||||
logf("%s is unanswered since %s and its callee's machine is heard: the finding %s", callee, since.UTC().Format(time.RFC3339), key)
|
||||
}
|
||||
}
|
||||
|
||||
// endedAtBind is readEndedAtBind's one read: the held asks, the seat's verb (a kept call whose wait is kept, wait), and
|
||||
// each ended asking written again.
|
||||
func endedAtBind(ctx context.Context, js jetstream.JetStream, asksKV jetstream.KeyValue, stale staleOf, bus caller, wait callWait, created, now time.Time, logf func(string, ...any)) error {
|
||||
keys, err := asksKV.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held := map[string][]byte{}
|
||||
for _, key := range keys {
|
||||
if e, err := asksKV.Get(ctx, key); err == nil {
|
||||
held[key] = e.Value()
|
||||
}
|
||||
}
|
||||
query, ok := assign.ChannelAsksQuery(held)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
answer, err := wait.keep(ctx, asksCallID, channelAsksVerb, func() (json.RawMessage, error) { return bus.CallKept(channelAsksVerb, query) }, logf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended, cut, err := assign.EndedAtChannel(held, answer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cut != "" {
|
||||
logf("%s at the bind: %s", channelAsksVerb, cut)
|
||||
}
|
||||
var putsKV jetstream.KeyValue
|
||||
if js != nil {
|
||||
if kv, err := js.KeyValue(ctx, buckets.Puts()); err == nil {
|
||||
putsKV = kv
|
||||
}
|
||||
}
|
||||
for _, e := range ended {
|
||||
body, err := tellerevents.EncodeEnded(e)
|
||||
if err == nil {
|
||||
err = askAgain(ctx, asksKV, putsKV, stale, body, now, logf)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
logf("%s at the bind of %s: %d of %d held asks ended at the channel without an answer", channelAsksVerb, created.Format(time.RFC3339), len(ended), len(held))
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgain writes the ask the teller said ended again under a new asking, by compare-and-set at the revision that holds the
|
||||
// asking that ended (assign.AskedAgain), and says in the journal what it wrote or why it wrote nothing. Only a bus that
|
||||
// could not be read or written is an error, so the event is heard again; a body that does not read is said and dropped.
|
||||
func askAgain(ctx context.Context, asksKV, putsKV jetstream.KeyValue, stale staleOf, data []byte, now time.Time, logf func(string, ...any)) error {
|
||||
e, err := tellerevents.DecodeEnded(data)
|
||||
if err != nil {
|
||||
logf("the teller's %s holds what is not an ended ask, and is dropped: %v", tellerevents.AskEnded, err)
|
||||
return nil
|
||||
}
|
||||
if asksKV == nil {
|
||||
return errors.New("the assigner's asks are not open")
|
||||
}
|
||||
var current []byte
|
||||
var rev uint64
|
||||
entry, err := asksKV.Get(ctx, e.Key)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
current, rev = entry.Value(), entry.Revision()
|
||||
}
|
||||
var put *puts.Put
|
||||
if k, err := asks.ParseAnyAskKey(e.Key); err == nil && k.Kind == asks.AskKindRecord && putsKV != nil {
|
||||
if pkey, err := puts.Key(k.Node, k.Module); err == nil {
|
||||
pe, err := putsKV.Get(ctx, pkey)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if p, err := puts.Decode(pe.Value()); err == nil {
|
||||
put = &p
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
value, why := assign.AskedAgain(e, current, put, now)
|
||||
if why == "" && stale != nil {
|
||||
// An ask the pass takes away is not written again: it is about a move or a composition that has been left behind, and the
|
||||
// pass removes it (issue 1966 note 165).
|
||||
if s := stale(e.Key); s != "" {
|
||||
why = "it is stale and left to the pass to take away: " + s
|
||||
}
|
||||
}
|
||||
if why != "" {
|
||||
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is not written again: %s", e.Key, e.Ask, e.Outcome, why)
|
||||
return nil
|
||||
}
|
||||
if _, err := asksKV.Update(ctx, e.Key, value, rev); err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
logf("%s: the ask %s ended (%s) without an answer, and the key changed since: it is not written again", e.Key, e.Ask, e.Outcome)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
logf("%s: the ask %s ended (%s) at the operator channel without an answer, and is written again under a new asking, which the teller asks", e.Key, e.Ask, e.Outcome)
|
||||
return nil
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// Which proposals the assigner asks of the operator is its setting ask-operator-for (design 47 G A-425, A-425a): a list of change
|
||||
// kinds, set at mesh level with mesh-config's config, its default take-off, remove-node and stops. A proposal none of whose changes
|
||||
// is listed is applied at once when its verdict passes, said in the log and by assignments-changed; one with a listed kind is
|
||||
// asked as before. The setting is read from the file the manifest declares with the content ${config:ask-operator-for}, which the
|
||||
// environment names (A-112), at every pass, and a change of it is said.
|
||||
|
||||
// KindStops is the change kind of a verdict whose answers say a node would stop running a module.
|
||||
const KindStops = "stops"
|
||||
|
||||
// defaultAskOperatorFor is the setting's default.
|
||||
var defaultAskOperatorFor = []string{mmac.OpTakeOff, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// askKinds are the kinds the setting may name.
|
||||
var askKinds = []string{mmac.OpPlace, mmac.OpTakeOff, mmac.OpPin, mmac.OpUnpin, mmac.OpHold, mmac.OpUnhold, mmac.OpDecide, mmac.OpConfig,
|
||||
mmac.OpAddNode, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// parseAskOperatorFor reads the setting's value: kinds separated by commas or spaces; empty is the default. A kind it does not
|
||||
// know is answered apart and named nowhere in the list.
|
||||
func parseAskOperatorFor(raw string) (kinds, unknown []string) {
|
||||
fields := strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
if len(fields) == 0 {
|
||||
return slices.Clone(defaultAskOperatorFor), nil
|
||||
}
|
||||
for _, f := range fields {
|
||||
f = strings.ToLower(strings.TrimSpace(f))
|
||||
switch {
|
||||
case !slices.Contains(askKinds, f):
|
||||
unknown = append(unknown, f)
|
||||
case !slices.Contains(kinds, f):
|
||||
kinds = append(kinds, f)
|
||||
}
|
||||
}
|
||||
slices.Sort(kinds)
|
||||
return kinds, unknown
|
||||
}
|
||||
|
||||
// readAskOperatorFor reads the setting's file: the default for no path or no file.
|
||||
func readAskOperatorFor(path string) (kinds, unknown []string, err error) {
|
||||
if path == "" {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
k, u := parseAskOperatorFor(string(raw))
|
||||
return k, u, nil
|
||||
}
|
||||
|
||||
// askOperatorFor is the setting as this pass read it: the file read again each pass, a change said once, a file that cannot be
|
||||
// read keeping what was read last (the default before any).
|
||||
func (p *passer) askOperatorFor() []string {
|
||||
kinds, unknown, err := readAskOperatorFor(p.askForFile)
|
||||
if err != nil {
|
||||
if p.askFor == nil {
|
||||
p.askFor = slices.Clone(defaultAskOperatorFor)
|
||||
}
|
||||
p.logf("the setting ask-operator-for cannot be read from %s, so %s stands: %v", p.askForFile, strings.Join(p.askFor, ", "), err)
|
||||
return p.askFor
|
||||
}
|
||||
if !slices.Equal(kinds, p.askFor) {
|
||||
words := "the setting ask-operator-for is " + strings.Join(kinds, ", ") + ": a proposal with a change of another kind only is applied with no ask"
|
||||
if len(unknown) > 0 {
|
||||
words += "; kinds it names that are none are left out: " + strings.Join(unknown, ", ")
|
||||
}
|
||||
p.logf("%s", words)
|
||||
p.askFor = kinds
|
||||
}
|
||||
return p.askFor
|
||||
}
|
||||
|
||||
// asksFor says whether a proposal of these change kinds is asked of the operator.
|
||||
func (p *passer) asksFor(kinds []string) bool {
|
||||
set := p.askOperatorFor()
|
||||
for _, k := range kinds {
|
||||
if slices.Contains(set, k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// proposalKinds are the change kinds of a proposal whose verdict is v: each change's op, and stops when an answer says a node
|
||||
// would stop running a module.
|
||||
func proposalKinds(prop mma.Proposal, v mma.Verdict) []string {
|
||||
var out []string
|
||||
for _, c := range prop.Changes {
|
||||
if c.Op != "" && !slices.Contains(out, c.Op) {
|
||||
out = append(out, c.Op)
|
||||
}
|
||||
}
|
||||
if len(judge.Stops(v)) > 0 {
|
||||
out = append(out, KindStops)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// AssignerModule is the asker the assigner's compose asks name.
|
||||
const AssignerModule = "mesh-delivery-assigner"
|
||||
|
||||
// composeAsks are the compose asks the assigner put on the Go build seat's check stream (novox/hq ADR 0482 clause 4,
|
||||
// issues 1622 and 1623), by what each composes: one ask per key, put once and followed by a watch of its step key until
|
||||
// it ends, which wakes a pass. No timer: an ask that waits in the stream waits (its backlog is the view's and the
|
||||
// watchman's to show). A restart forgets them and asks again; the build module keeps a composer built for a commit, so a
|
||||
// second ask builds nothing.
|
||||
type composeAsks struct {
|
||||
ctx context.Context
|
||||
asker buildrun.Asker
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
asks map[string]*composeAsk
|
||||
}
|
||||
|
||||
type composeAsk struct {
|
||||
id string
|
||||
cancel context.CancelFunc
|
||||
ran *assign.Ran
|
||||
}
|
||||
|
||||
func newComposeAsks(ctx context.Context, asker buildrun.Asker, wake func(), logf func(string, ...any)) *composeAsks {
|
||||
return &composeAsks{ctx: ctx, asker: asker, wake: wake, logf: logf, now: time.Now, asks: map[string]*composeAsk{}}
|
||||
}
|
||||
|
||||
// get is the end of the ask that composes key, or nil and its id while it runs. The first get of a key puts the ask.
|
||||
// A job its layout refuses ends at once, failed with why: never silent.
|
||||
func (a *composeAsks) get(key string, job delivery.ComposeJob) (*assign.Ran, string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if k, ok := a.asks[key]; ok {
|
||||
return k.ran, k.id
|
||||
}
|
||||
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: AssignerModule,
|
||||
At: a.now(), Compose: &job}
|
||||
ctx, cancel := context.WithCancel(a.ctx)
|
||||
k := &composeAsk{id: ask.ID, cancel: cancel}
|
||||
a.asks[key] = k
|
||||
if err := a.asker.Put(ctx, ask); err != nil {
|
||||
a.logf("the compose ask %s for %s could not be put: %v", ask.ID, key, err)
|
||||
k.ran = &assign.Ran{State: delivery.StepFailed, Reason: "the compose ask could not be put on the Go build seat: " + err.Error()}
|
||||
cancel()
|
||||
return k.ran, k.id
|
||||
}
|
||||
go func() {
|
||||
step, err := a.asker.Wait(ctx, ask)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
ran := assign.RanOf(step)
|
||||
if err != nil {
|
||||
ran = assign.Ran{State: delivery.StepFailed, Reason: "the compose ask's step could not be followed: " + err.Error()}
|
||||
}
|
||||
a.mu.Lock()
|
||||
k.ran = &ran
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}()
|
||||
return nil, k.id
|
||||
}
|
||||
|
||||
// sweep forgets, and stops following, every ask whose key no pass asked for this time: a candidate gone or moved on.
|
||||
func (a *composeAsks) sweep(seen map[string]bool) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for key, k := range a.asks {
|
||||
if !seen[key] {
|
||||
k.cancel()
|
||||
delete(a.asks, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeAtHead is the composition of a group with a mesh-sdk member: the composer at its head and at each reader's
|
||||
// mesh-sdk, asked of the Go build seat, and the verdict once every ask ended; until then the verdict waits, naming the
|
||||
// asks. The keys it used are marked in seen.
|
||||
func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Change, seen map[string]bool, began time.Time) delivery.CompositionVerdict {
|
||||
waiting := func(words string) delivery.CompositionVerdict {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionWaits, Words: words}
|
||||
}
|
||||
head, err := assign.HeadJob(m, ch)
|
||||
if err != nil {
|
||||
return delivery.CompositionVerdict{Group: ch.Group, Heads: ch.Heads, State: delivery.CompositionFail, SDK: "mesh-sdk " + ch.SDKHead,
|
||||
Words: "the compose ask at the group's mesh-sdk head cannot be made: " + err.Error()}
|
||||
}
|
||||
var pending []string
|
||||
key := "head." + ch.Group + "." + ch.Heads + "." + ch.SDKHead
|
||||
seen[key] = true
|
||||
headRan, id := p.asks.get(key, head)
|
||||
if headRan == nil {
|
||||
pending = append(pending, "mesh-sdk "+ch.SDKHead+" (ask "+id+")")
|
||||
}
|
||||
var readers []assign.Reader
|
||||
var notJudged []string
|
||||
for _, module := range sortedNames(assign.ReaderModules) {
|
||||
repository := assign.ReaderModules[module]
|
||||
var commits []string
|
||||
member := false
|
||||
if h := ch.Members[repository]; h != "" {
|
||||
commits, member = []string{h}, true
|
||||
} else {
|
||||
commits = p.w.runningCommits(module)
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+
|
||||
"trunk build of it, and the group holds no member of "+repository)
|
||||
continue
|
||||
}
|
||||
for _, commit := range commits {
|
||||
job, err := assign.ReaderJob(ch, module, repository, commit)
|
||||
if err != nil {
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member,
|
||||
Ran: assign.Ran{State: delivery.StepFailed, Reason: err.Error()}})
|
||||
continue
|
||||
}
|
||||
rkey := "reader." + ch.Group + "." + ch.Heads + "." + module + "." + commit
|
||||
seen[rkey] = true
|
||||
ran, rid := p.asks.get(rkey, job)
|
||||
if ran == nil {
|
||||
pending = append(pending, module+" at "+commit+" (ask "+rid+")")
|
||||
continue
|
||||
}
|
||||
readers = append(readers, assign.Reader{Module: module, Commit: commit, Member: member, Ran: *ran})
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
return waiting("composing on the Go build seat with the mesh-sdk of " + strings.Join(pending, ", "))
|
||||
}
|
||||
v := assign.FromComposer(m, ch, *headRan, readers, time.Since(began))
|
||||
v.Warnings = append(v.Warnings, notJudged...)
|
||||
return v
|
||||
}
|
||||
|
||||
func sortedNames(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks
|
||||
// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps
|
||||
// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first.
|
||||
|
||||
const (
|
||||
asksTool = "assigner_asks"
|
||||
askDropTool = "assigner_ask_drop"
|
||||
)
|
||||
|
||||
// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total).
|
||||
const asksAnswerMost = 768 << 10
|
||||
|
||||
// takenReader is what assigner_asks reads of the state taken (stdio.KeptState).
|
||||
type takenReader interface {
|
||||
Keys() ([]string, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
}
|
||||
|
||||
// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.<ask>) and the drops of
|
||||
// asks not pulled yet (dropped.<ask>), each list cut at the answer's bound with capped and total.
|
||||
func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) {
|
||||
w, err := waiting(meshwants.Seat)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err)
|
||||
}
|
||||
keys, err := taken.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
// An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a
|
||||
// nak (H-A-410's addenda): the bus says what it knows, the state taken the rest.
|
||||
size := 0
|
||||
waits := make([]map[string]any, 0, len(w.Asks))
|
||||
for _, a := range w.Asks {
|
||||
size += len(a.ID) + len(a.Module) + 64
|
||||
state := "waiting"
|
||||
if a.Again {
|
||||
state = "waiting again"
|
||||
if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) {
|
||||
state = "held"
|
||||
}
|
||||
}
|
||||
waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state})
|
||||
}
|
||||
takenIDs, dropped := []string{}, []map[string]any{}
|
||||
total, capped := 0, w.Capped
|
||||
for _, k := range keys {
|
||||
if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok {
|
||||
total++
|
||||
if size += len(id) + 4; size > asksAnswerMost {
|
||||
capped = true
|
||||
continue
|
||||
}
|
||||
takenIDs = append(takenIDs, id)
|
||||
continue
|
||||
}
|
||||
id, ok := strings.CutPrefix(k, droppedPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
total++
|
||||
e, err := taken.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
var d askDrop
|
||||
if json.Unmarshal(e.Value, &d) != nil {
|
||||
continue
|
||||
}
|
||||
if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost {
|
||||
capped = true
|
||||
continue
|
||||
}
|
||||
dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At})
|
||||
}
|
||||
return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil
|
||||
}
|
||||
|
||||
// asksTools are the module's tools of the seat's work queue.
|
||||
func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " +
|
||||
"that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " +
|
||||
"cut at 768 KB with capped and total. Read-only.",
|
||||
Input: schema(map[string]string{}, nil),
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return asksOf(waiting, taken)
|
||||
}},
|
||||
{Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " +
|
||||
"first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " +
|
||||
"its step could not run naming the drop and who asked it.",
|
||||
Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why"))
|
||||
if id == "" || why == "" {
|
||||
return nil, errors.New("name the ask and why it is ended")
|
||||
}
|
||||
by := stdio.CallerOf(args)
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err)
|
||||
}
|
||||
out["hand-act"] = taken.Log
|
||||
return out, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat
|
||||
// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process.
|
||||
const keepNowTool = "assigner_keep_now"
|
||||
|
||||
func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " +
|
||||
"hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " +
|
||||
"the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).",
|
||||
Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
why := strings.TrimSpace(arg(args, "why"))
|
||||
if why == "" {
|
||||
return nil, errors.New("say why the keeping runs now")
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err)
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err)
|
||||
}
|
||||
out["hand-act"] = taken.Log
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G §717: the assigner's asks are the seat mesh-wants' state asks, so every key its asks package writes is one of
|
||||
// that state's key forms, read back to the same parts, and each value is the seat's Ask.
|
||||
func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
|
||||
const hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
const fp, digest = "src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
must := func(k string, err error) string {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
three := func(parse func(string) (string, string, string, bool)) func(string) []string {
|
||||
return func(k string) []string {
|
||||
if a, b, c, ok := parse(k); ok {
|
||||
return []string{a, b, c}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
key string
|
||||
parse func(string) []string
|
||||
want []string
|
||||
}{
|
||||
{must(asks.HoldAskKey("web", fp)), func(k string) []string {
|
||||
if a, b, ok := meshwants.ParseAsksByHoldModuleFingerprintKey(k); ok {
|
||||
return []string{a, b}
|
||||
}
|
||||
return nil
|
||||
}, []string{"web", hex}},
|
||||
{must(asks.RestoreAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRestoreNodeModuleFingerprintKey), []string{"ace", "web", hex}},
|
||||
{must(asks.BackupAskKey("ace", "web", fp)), three(meshwants.ParseAsksByBackupNodeModuleFingerprintKey), []string{"ace", "web", hex}},
|
||||
{must(asks.OutsideAskKey("ace", "web", fp)), three(meshwants.ParseAsksByOutsideNodeModuleFingerprintKey), []string{"ace", "web", hex}},
|
||||
{must(asks.RevertAskKey("ace", "web", fp)), three(meshwants.ParseAsksByRevertNodeModuleFingerprintKey), []string{"ace", "web", hex}},
|
||||
{must(asks.RecordAskKey("ace", "web", digest)), three(meshwants.ParseAsksByRecordNodeModuleDigestKey), []string{"ace", "web", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}},
|
||||
{must(asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")), func(k string) []string {
|
||||
if a, ok := meshwants.ParseAsksByAssignmentsProposalKey(k); ok {
|
||||
return []string{a}
|
||||
}
|
||||
return nil
|
||||
}, []string{"p-20261028t183000-1a2b3c"}},
|
||||
} {
|
||||
got := row.parse(row.key)
|
||||
if len(got) != len(row.want) {
|
||||
t.Errorf("%s is no key form of mesh-wants' asks: read as %v", row.key, got)
|
||||
continue
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != row.want[i] {
|
||||
t.Errorf("%s reads as %v, want %v", row.key, got, row.want)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
key := must(asks.HoldAskKey("web", fp))
|
||||
raw, err := asks.EncodeAsk(key, asks.Ask{Ask: json.RawMessage(`{"id":"x"}`), Asking: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
|
||||
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the
|
||||
// builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's
|
||||
// `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard.
|
||||
|
||||
// builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded).
|
||||
type builtEnds struct {
|
||||
mu sync.Mutex
|
||||
ends map[string]builds.BuildEnded
|
||||
}
|
||||
|
||||
func (b *builtEnds) key(module, fp string) string { return module + "@" + fp }
|
||||
|
||||
// put takes one build.ended in.
|
||||
func (b *builtEnds) put(data []byte) error {
|
||||
var e builds.BuildEnded
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return fmt.Errorf("a build.ended that is no end: %w", err)
|
||||
}
|
||||
if e.Module == "" || e.Fingerprint == "" {
|
||||
return fmt.Errorf("a build's end naming no module or fingerprint: %s", data)
|
||||
}
|
||||
if e.Outcome == "" { // an end said before A-162 is a done build's
|
||||
e.Outcome = builds.BuildDone
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.ends == nil {
|
||||
b.ends = map[string]builds.BuildEnded{}
|
||||
}
|
||||
b.ends[b.key(e.Module, e.Fingerprint)] = e
|
||||
return nil
|
||||
}
|
||||
|
||||
// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended.
|
||||
func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
e, ok := b.ends[b.key(module, fp)]
|
||||
return e, ok
|
||||
}
|
||||
|
||||
// partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `<module>:<artifact>`), with the
|
||||
// node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose
|
||||
// parts the version's release says (releasedVersions).
|
||||
func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) {
|
||||
b.mu.Lock()
|
||||
e, ok := b.ends[b.key(module, fp)]
|
||||
b.mu.Unlock()
|
||||
if !ok || (e.KeptOn == "" && e.Branch == "") {
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for name, digest := range e.Made {
|
||||
parts[module+":"+name] = digest
|
||||
}
|
||||
return assign.KeptBuild{Parts: parts, KeptOn: e.KeptOn}, true
|
||||
}
|
||||
|
||||
// releasedVersions are the catalog's version-released heard, by module and source revision.
|
||||
type releasedVersions struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]assign.KeptBuild
|
||||
// newest is, by module, the last version-released heard of it (delivered.go).
|
||||
newest map[string]releaseHeard
|
||||
}
|
||||
|
||||
func (r *releasedVersions) key(module string, source int64) string {
|
||||
return fmt.Sprintf("%s@%d", module, source)
|
||||
}
|
||||
|
||||
// put takes one version-released in.
|
||||
func (r *releasedVersions) put(data []byte) error {
|
||||
var e meshcatalog.VersionReleasedEvent
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return fmt.Errorf("a version-released that is none: %w", err)
|
||||
}
|
||||
if e.Module == "" || e.Version == "" {
|
||||
return fmt.Errorf("a version-released naming no module or version: %s", data)
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for _, a := range e.Artifacts {
|
||||
parts[partRef(e.Module, a.Artifact)] = a.Digest
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.seen == nil {
|
||||
r.seen = map[string]assign.KeptBuild{}
|
||||
}
|
||||
r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts}
|
||||
if r.newest == nil {
|
||||
r.newest = map[string]releaseHeard{}
|
||||
}
|
||||
h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source}
|
||||
if e.At != nil {
|
||||
h.At = *e.At
|
||||
}
|
||||
r.newest[e.Module] = h // the stream's order is the order of release
|
||||
return nil
|
||||
}
|
||||
|
||||
// withParts is the version of module whose release named exactly parts (`<module>:<artifact>` -> digest), the newest such when
|
||||
// more than one did; false when none did or parts is empty (design 47 G A-421).
|
||||
func (r *releasedVersions) withParts(module string, parts map[string]string) (assign.KeptBuild, bool) {
|
||||
if len(parts) == 0 {
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out assign.KeptBuild
|
||||
found := false
|
||||
for key, k := range r.seen {
|
||||
if m, _, ok := strings.Cut(key, "@"); !ok || m != module || !maps.Equal(k.Parts, parts) {
|
||||
continue
|
||||
}
|
||||
if !found || compareVersion(k.Version, out.Version) > 0 {
|
||||
out, found = k, true
|
||||
}
|
||||
}
|
||||
return out, found
|
||||
}
|
||||
|
||||
// madeOf is what module's done build at fp made, by `<module>:<artifact>`, as its build.ended names it; none for a build not heard
|
||||
// or not done (design 47 G A-421).
|
||||
func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
e, ok := b.endOf(module, fp)
|
||||
if !ok || e.Outcome != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for name, digest := range e.Made {
|
||||
parts[module+":"+name] = digest
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
k, ok := r.seen[r.key(module, source)]
|
||||
return k, ok
|
||||
}
|
||||
|
||||
// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and
|
||||
// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703).
|
||||
var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream)
|
||||
|
||||
// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was
|
||||
// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before
|
||||
// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner
|
||||
// starts again.
|
||||
func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) {
|
||||
acct := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
nh := notHeardWriter(findings, logf)
|
||||
cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) {
|
||||
if err := ends.put(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf, process.FromFirst()) // read whole from its first message whether the bus keeps it last per subject or by age (A-181, A-233)
|
||||
if err != nil {
|
||||
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
|
||||
return
|
||||
}
|
||||
nh.Heard(buildsStreamName)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
// compareVersion orders two MAJOR.MINOR.PATCH versions by their numbers; a part that is no number counts as 0.
|
||||
func compareVersion(a, b string) int {
|
||||
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; i < 3; i++ {
|
||||
var x, y int
|
||||
if i < len(pa) {
|
||||
x, _ = strconv.Atoi(pa[i])
|
||||
}
|
||||
if i < len(pb) {
|
||||
y, _ = strconv.Atoi(pb[i])
|
||||
}
|
||||
if x != y {
|
||||
return x - y
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// partRef is a release's artifact as a part's key, `<module>:<artifact>`: the contract's VersionArtifact names the artifact
|
||||
// reference whole (an ArtifactRef), as the catalog's version-released says it; a bare name is taken as the module's (A-96).
|
||||
func partRef(module, artifact string) string {
|
||||
if strings.Contains(artifact, ":") {
|
||||
return artifact
|
||||
}
|
||||
return module + ":" + artifact
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// Design 47 G §379, §526, A-96: a build.ended heard names a kept build's or another branch's parts by digest (`<module>:<artifact>`) and its
|
||||
// keeping node; a build of the trunk not kept names none, its parts being the release's.
|
||||
func TestABuildEndedHeardNamesAKeptBuildsPartsAndItsNode(t *testing.T) {
|
||||
put := func(b *builtEnds, e builds.BuildEnded) {
|
||||
raw, _ := json.Marshal(e)
|
||||
if err := b.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
made := map[string]string{"program": "sha256:1"}
|
||||
var b builtEnds
|
||||
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "a", Made: made})
|
||||
if _, ok := b.partsOf("node-engine", "a"); ok {
|
||||
t.Error("a trunk build not kept names parts")
|
||||
}
|
||||
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "b", Made: made, KeptOn: "ace"})
|
||||
if k, ok := b.partsOf("node-engine", "b"); !ok || k.KeptOn != "ace" || !reflect.DeepEqual(k.Parts, map[string]string{"node-engine:program": "sha256:1"}) {
|
||||
t.Errorf("kept build = %+v, %v", k, ok)
|
||||
}
|
||||
put(&b, builds.BuildEnded{Module: "node-engine", Fingerprint: "c", Made: made, Branch: "cand"})
|
||||
if k, ok := b.partsOf("node-engine", "c"); !ok || k.KeptOn != "" || len(k.Parts) != 1 {
|
||||
t.Errorf("branch build = %+v, %v", k, ok)
|
||||
}
|
||||
if err := b.put([]byte(`{"module":"x"}`)); err == nil {
|
||||
t.Error("an end naming no fingerprint is kept")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §379, A-96: a version-released heard names the version and the parts' digests of the module's source revision.
|
||||
func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
|
||||
var r releasedVersions
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
|
||||
if err := r.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := assign.KeptBuild{Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}}
|
||||
if k, ok := r.of("mail", 7); !ok || !reflect.DeepEqual(k, want) {
|
||||
t.Errorf("released = %+v, %v", k, ok)
|
||||
}
|
||||
if _, ok := r.of("mail", 8); ok {
|
||||
t.Error("another source revision names a version")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §379, A-96, A-421; H §764, H-A-48: the trunk's entry, which the catalog stamps with no source, takes its version and
|
||||
// parts from the version-released whose parts are what its build.ended made; a kept build from its end; no revision kept, another
|
||||
// fingerprint and a build whose parts no release names name none, and an unheard release is said once.
|
||||
func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsBuildsParts(t *testing.T) {
|
||||
w := newWorld()
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "k", Made: map[string]string{"image": "sha256:2"}, KeptOn: "g14"})
|
||||
if err := w.ends.put(endRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var logs []string
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}, logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
named := p.namedBuild("mail", "t")
|
||||
if _, ok := named("t"); ok {
|
||||
t.Error("a trunk entry whose build.ended is not heard names a version")
|
||||
}
|
||||
_, _ = named("t")
|
||||
if k, ok := named("k"); !ok || k.KeptOn != "g14" || k.Version != "" {
|
||||
t.Errorf("kept build = %+v, %v", k, ok)
|
||||
}
|
||||
if _, ok := named("other"); ok {
|
||||
t.Error("another fingerprint names a release")
|
||||
}
|
||||
// A build held back at an older fingerprint names the release whose parts are exactly what it made.
|
||||
heldRaw, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "old", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(heldRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k, ok := named("old"); !ok || k.Version != "1.2.3" {
|
||||
t.Errorf("a held build whose parts were released = %+v, %v; want 1.2.3", k, ok)
|
||||
}
|
||||
// The entry's module value carries no source stamp: the catalog writes none (H-A-48).
|
||||
if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{}}); v.Fp2 != "t" {
|
||||
t.Errorf("the entry's module value is %+v", v)
|
||||
}
|
||||
made, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "t", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(made); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.graph.source = 8 // a later merge that did not rebuild mail
|
||||
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" {
|
||||
t.Errorf("the trunk entry whose build made the released digests: %+v, %v", k, ok)
|
||||
}
|
||||
if len(logs) != 1 || !strings.Contains(logs[0], "the release of mail at fingerprint t is not heard: no version-released of it names the parts its build.ended made") {
|
||||
t.Errorf("logs = %q, want the unheard release said once, naming the module and the fingerprint", logs)
|
||||
}
|
||||
p.graph.ok = false
|
||||
if _, ok := p.namedBuild("mail", "t")("t"); ok {
|
||||
t.Error("no revision kept names a release")
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
)
|
||||
|
||||
// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G
|
||||
// A-181: what was built is heard, never read from the builder's bucket).
|
||||
func endDone(t testing.TB, w *world, module, fp, manifest string) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.ends.put(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507).
|
||||
const trunkBranch = "main"
|
||||
@@ -1,72 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// addedOf is the `added` of node's candidate (design 47 G A-316, §379, §421): by module, each module the candidate adds to node
|
||||
// (laid over current) with no want there, the version and parts the want it would write carries, taken from the same code that
|
||||
// writes a want's (namedBuild, the want's build at the fingerprint of the branch the want's code takes). A module whose build names none is not listed.
|
||||
func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]mma.Assignments) map[string]mma.AddedModule {
|
||||
if laid == nil {
|
||||
return nil
|
||||
}
|
||||
placed := map[string]string{}
|
||||
for _, m := range laid.Modules {
|
||||
placed[m.Module] = m.Branch
|
||||
}
|
||||
var out map[string]mma.AddedModule
|
||||
for _, module := range plc.ChangeOf(current[node], *laid).Added {
|
||||
if key, err := mwc.WantKey(node, module); err != nil || p.w.has(buckets.Wants(), key) {
|
||||
continue
|
||||
}
|
||||
// The branch the want's code takes: the one the candidate's placement names, else the module's default (groups, wants).
|
||||
branch := p.w.trunkName()
|
||||
if p.ends.following() {
|
||||
branch = p.ends.branchOf(module, placed[module])
|
||||
}
|
||||
entry := p.w.trunkOf(branch)[module]
|
||||
fp := entry.Fp2
|
||||
if fp == "" {
|
||||
p.skippedAdd(node, module, "", "the branch "+branch+" names no fingerprint of it")
|
||||
continue
|
||||
}
|
||||
k, ok := p.namedBuild(module, fp)(fp)
|
||||
if !ok {
|
||||
p.skippedAdd(node, module, fp, "no version-released of it is heard for that fingerprint, so the candidate carries no version and no parts")
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]mma.AddedModule{}
|
||||
}
|
||||
added := mma.AddedModule{Version: k.Version, Parts: k.Parts}
|
||||
if k.Version != "" {
|
||||
added.Parts = nil // a released build is named by its version alone (design 47 G A-430, A-430a)
|
||||
}
|
||||
out[module] = added
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// candidateOf is the candidate of proposal id for node: the assignments laid over it, and what it adds (addedOf).
|
||||
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
|
||||
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
|
||||
}
|
||||
|
||||
// skippedAdd says once per node, module and fingerprint why a module the candidate adds is not listed in `added`.
|
||||
func (p *passer) skippedAdd(node, module, fp, why string) {
|
||||
key := node + "/" + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the candidate adds %s to %s with no parts: %s (fingerprint %q)", module, node, why, fp)
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G A-316: the candidate of a node carries, for each module it adds there without a want, the version and parts the
|
||||
// want it would write carries; a module with a want there, or a build that names none, is not listed.
|
||||
func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
|
||||
w := newWorld()
|
||||
for _, m := range []string{"mail", "web", "db"} {
|
||||
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
|
||||
}
|
||||
w.first = "main"
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(rel); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
built, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-mail", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
end, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: "fp-db", Made: map[string]string{"program": "sha256:3"}, KeptOn: "g14"})
|
||||
if err := w.ends.put(end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 9}} // a later merge: the release is matched by the parts its build made (A-421, H-A-48)
|
||||
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
|
||||
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
|
||||
c := p.candidateOf("p1", "ace", laid, current)
|
||||
want := map[string]mma.AddedModule{
|
||||
"mail": {Version: "1.2.3"}, // a released build is named by its version alone (A-430a)
|
||||
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
|
||||
}
|
||||
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
|
||||
t.Errorf("candidate = %+v, added want %+v", c, want)
|
||||
}
|
||||
if got := p.candidateOf("p1", "ace", nil, current); got.Added != nil {
|
||||
t.Errorf("a candidate taking the key away adds %+v", got.Added)
|
||||
}
|
||||
// A placement on a watched branch is added at that branch's fingerprint, as the want's code takes it, not the trunk's.
|
||||
w.putModule("feat", "mail", meshcatalog.ModulesByModuleValue{Fp2: "fp-feat"}, graph.Node{}, nil)
|
||||
feat, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "fp-feat", Made: map[string]string{"program": "sha256:4"}, KeptOn: "g14"})
|
||||
if err := w.ends.put(feat); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = w.trunkName
|
||||
p.ends.on.Store(true)
|
||||
onFeat := &mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "mail", Branch: "feat"}}}
|
||||
if got, want := p.candidateOf("p2", "g14", onFeat, nil).Added, map[string]mma.AddedModule{"mail": {Parts: map[string]string{"mail:program": "sha256:4"}}}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("a placement on branch feat adds %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-<id>`'s catalog graph revision,
|
||||
// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is
|
||||
// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate.<group>.<heads>;
|
||||
// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing.
|
||||
|
||||
// runGroupPrefix is a run on request's candidate group before its id, `run-<id>` (A-160).
|
||||
const runGroupPrefix = "run-"
|
||||
|
||||
// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the
|
||||
// candidate's group and heads and the revision its event names.
|
||||
func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) {
|
||||
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
after := ""
|
||||
for {
|
||||
a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after})
|
||||
if err != nil {
|
||||
return half, err
|
||||
}
|
||||
if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group {
|
||||
return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision)
|
||||
}
|
||||
half.Revision = *a.Revision
|
||||
for _, e := range a.Entries {
|
||||
half.Entries[e.Module] = e
|
||||
}
|
||||
if a.Next == "" {
|
||||
return half, nil
|
||||
}
|
||||
after = a.Next
|
||||
}
|
||||
}
|
||||
|
||||
// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a
|
||||
// group's) is passed over.
|
||||
func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error {
|
||||
if !strings.HasPrefix(ev.Group, runGroupPrefix) {
|
||||
return nil
|
||||
}
|
||||
g := p.graph
|
||||
if g == nil || g.objects == nil || g.halves == nil {
|
||||
return errors.New("the assigner keeps no assignment graph, so no candidate's is kept")
|
||||
}
|
||||
half, err := readCandidateHalf(call, ev)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err)
|
||||
}
|
||||
in := p.graphInputs()
|
||||
in.Halves = maps.Clone(in.Halves)
|
||||
in.Halves[half.Revision.Branch] = half
|
||||
c := assigngraph.Compute(in)
|
||||
for _, u := range c.Uncomputed {
|
||||
p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u)
|
||||
}
|
||||
rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err)
|
||||
}
|
||||
if fresh {
|
||||
p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235).
|
||||
func (p *passer) dropCandidate(ctx context.Context, group string) error {
|
||||
g := p.graph
|
||||
if g == nil || g.objects == nil {
|
||||
return nil
|
||||
}
|
||||
o, ok := g.objects.(assigngraph.Remover)
|
||||
if !ok {
|
||||
return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group)
|
||||
}
|
||||
n, err := assigngraph.DropCandidate(ctx, o, group)
|
||||
if err == nil && n > 0 {
|
||||
p.logf("the candidate %s's assignment graph is dropped: its run ended", group)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it.
|
||||
func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
|
||||
raw, err := call(mcc.RevisionVerb, in)
|
||||
if err != nil {
|
||||
return meshcatalog.RevisionAnswer{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
|
||||
}
|
||||
var a meshcatalog.RevisionAnswer
|
||||
if err := json.Unmarshal(answer, &a); err != nil {
|
||||
return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
// and answers a compose ask by running mesh-sdk's composer over the request filled from the indexer's manifests (the
|
||||
// linked composer standing for the one the build module builds at the asked commit), writing its step on its node's key.
|
||||
type holder struct {
|
||||
mu sync.Mutex
|
||||
asks []delivery.StepAsk
|
||||
}
|
||||
|
||||
func (h *holder) taken() []delivery.StepAsk {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]delivery.StepAsk(nil), h.asks...)
|
||||
}
|
||||
|
||||
func (b *bed) hold(t *testing.T) *holder {
|
||||
t.Helper()
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
worker, _ := delivery.AskWorker(delivery.AskGo, delivery.AskCheck)
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
cons, err := b.js.CreateOrUpdateConsumer(b.ctx, seat, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := b.buckets[delivery.StepsBucket(goBuildModule)]
|
||||
h := &holder{}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
a, err := delivery.DecodeStepAsk(m.Data())
|
||||
if err != nil || a.Compose == nil {
|
||||
t.Errorf("the seat was put an ask that is not a compose: %v %s", err, m.Data())
|
||||
m.Ack()
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.asks = append(h.asks, a)
|
||||
h.mu.Unlock()
|
||||
req, err := composer.Decode(a.Compose.Request)
|
||||
if err == nil {
|
||||
err = composer.Fill(&req, func(digest string) ([]byte, error) {
|
||||
key, _ := delivery.ManifestsKey(digest)
|
||||
e, err := b.buckets[delivery.ManifestsBucket].Get(b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
man, err := delivery.DecodeManifest(e.Value())
|
||||
return []byte(man.JSON), err
|
||||
})
|
||||
}
|
||||
step := delivery.Step{ID: a.ID, Tool: "compose", Node: "ace", State: delivery.StepPassed, Started: time.Now().UTC(), Ended: time.Now().UTC()}
|
||||
if err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else if v, err := composer.Run(req); err != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
} else {
|
||||
vr, _ := json.Marshal(v)
|
||||
step.Result, _ = json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0123456789abcdef", Verdict: vr})
|
||||
}
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
t.Errorf("the step cannot be written: %v", err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
if _, err := steps.Put(b.ctx, key, raw); err != nil {
|
||||
t.Errorf("the step %s: %v", key, err)
|
||||
}
|
||||
m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(cc.Stop)
|
||||
return h
|
||||
}
|
||||
|
||||
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
|
||||
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
|
||||
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// head's composer was built with, and asks nothing more on a later pass of the same heads.
|
||||
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
running := strings.Repeat("3", 40)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.built(t, "mesh-controller", "novox/mesh-controller", running)
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
pulls := []delivery.CandidatePull{{Head: heads[0]}, {Head: heads[1],
|
||||
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||
b.start(t)
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
// No holder takes the asks yet: the composition waits, naming them.
|
||||
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
|
||||
}
|
||||
h := b.hold(t)
|
||||
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
|
||||
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14", v)
|
||||
}
|
||||
asks := h.taken()
|
||||
if len(asks) != 2 {
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
|
||||
}
|
||||
byRepo := map[string]delivery.StepAsk{}
|
||||
for _, a := range asks {
|
||||
byRepo[a.Compose.Fetch["repository"].(string)] = a
|
||||
if a.Asker != AssignerModule || a.Kind != delivery.AskCheck || a.Language != delivery.AskGo {
|
||||
t.Errorf("ask = %+v", a)
|
||||
}
|
||||
}
|
||||
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
|
||||
t.Errorf("the head's ask = %+v", a.Compose)
|
||||
}
|
||||
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running controller's ask = %+v", a.Compose)
|
||||
}
|
||||
// A later pass of the same heads (woken by any change) asks nothing more.
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := len(h.taken()); n != 2 {
|
||||
t.Errorf("a later pass of the same heads asked again: %d asks", n)
|
||||
}
|
||||
}
|
||||
|
||||
// verdict waits for the composition under key to satisfy ok, and answers it.
|
||||
func (b *bed) verdict(t *testing.T, key string, ok func(delivery.CompositionVerdict) bool) delivery.CompositionVerdict {
|
||||
t.Helper()
|
||||
var last delivery.CompositionVerdict
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
t.Fatalf("no composition under %s as wanted after 10 s; last %+v", key, last)
|
||||
return last
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
)
|
||||
|
||||
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
|
||||
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
|
||||
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
|
||||
// where they part. The assigner composes nothing and never writes these.
|
||||
|
||||
// publishedRow is one composition a node published: the module ("node" for its own), the fingerprint, the digest, or why
|
||||
// it was refused or cannot be read.
|
||||
type publishedRow struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Unread string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
|
||||
// names its node first): each value decoded and digested here, as the node-engine digests it (nodeengineclient.ComposedDigest).
|
||||
// A value or key that does not read is named with why, never left out.
|
||||
func published(values map[string][]byte) map[string][]publishedRow {
|
||||
byNode := map[string]map[string][]byte{}
|
||||
for key, raw := range values {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
if byNode[node] == nil {
|
||||
byNode[node] = map[string][]byte{}
|
||||
}
|
||||
byNode[node][key] = raw
|
||||
}
|
||||
out := map[string][]publishedRow{}
|
||||
for node, keys := range byNode {
|
||||
var rows []publishedRow
|
||||
for key, raw := range keys {
|
||||
row := publishedRow{Key: key}
|
||||
_, module, part, isNode, err := nodeengineclient.ParseComposedKey(key)
|
||||
var fp string
|
||||
if err == nil && !isNode {
|
||||
fp, err = catalogkey.ParseKeyPart(part)
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
row.Unread = err.Error()
|
||||
case isNode:
|
||||
row.Module = "node"
|
||||
default:
|
||||
row.Module, row.FP = module, fp
|
||||
}
|
||||
if err == nil {
|
||||
c, derr := nodeengineclient.DecodeComposition(raw)
|
||||
switch {
|
||||
case derr != nil:
|
||||
row.Unread = derr.Error()
|
||||
case c.Refused != "":
|
||||
row.Refused = c.Refused
|
||||
default:
|
||||
if d, err := nodeengineclient.ComposedDigest(c); err != nil {
|
||||
row.Unread = err.Error()
|
||||
} else {
|
||||
row.Digest = d
|
||||
}
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key })
|
||||
out[node] = rows
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
|
||||
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
|
||||
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
|
||||
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
|
||||
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
|
||||
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
|
||||
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
|
||||
|
||||
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
|
||||
const waitOnBuildNode = "build node"
|
||||
|
||||
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
|
||||
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
|
||||
|
||||
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
|
||||
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
|
||||
return func(ask, why string) error {
|
||||
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, said, ok := buscalls.Unwrap(raw); !ok {
|
||||
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
|
||||
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
|
||||
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
|
||||
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
|
||||
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
|
||||
if run.Started != nil {
|
||||
w.Since = *run.Started
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
|
||||
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
|
||||
if r.say == nil {
|
||||
return
|
||||
}
|
||||
for _, c := range readers {
|
||||
w := waitOf(run, c)
|
||||
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
|
||||
if how != "" {
|
||||
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
|
||||
}
|
||||
if err := r.say(event, w, id); err != nil {
|
||||
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
|
||||
func (r *compositionRuns) untaken(key, ask string) bool {
|
||||
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
|
||||
return err == nil && standing && held == ask
|
||||
}
|
||||
|
||||
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
|
||||
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(boundKey)
|
||||
if err != nil || raw == nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(raw, &v)
|
||||
return v.Ask, rev, true, nil
|
||||
}
|
||||
|
||||
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
|
||||
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
|
||||
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
|
||||
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
|
||||
rev, err := r.kv.arm(prefix+key, ask, ttl)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok && run.Ended == nil && run.Ask == ask {
|
||||
return true, nil
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
|
||||
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
|
||||
// ended, which moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
|
||||
key, run, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
|
||||
}
|
||||
if !armed {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.untaken(key, e.Ask) {
|
||||
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
r.sayWaits(run, run.Readers, "ran")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
|
||||
// late one moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
|
||||
key, _, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
armed := false
|
||||
if key != "" {
|
||||
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !armed {
|
||||
r.late("step-progress", e.Asker, e.Ask)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
|
||||
func (r *compositionRuns) late(signal, asker, ask string) {
|
||||
if asker == runAsker && ask != "" {
|
||||
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
|
||||
}
|
||||
}
|
||||
|
||||
// runOfAsk is the running run of an ask the assigner put, or none.
|
||||
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
|
||||
if asker != runAsker || ask == "" {
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(ask)
|
||||
if err != nil || key == "" || run.Ended != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
return key, run, nil
|
||||
}
|
||||
|
||||
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
|
||||
// ask and the bound. A run that ended already, or a key of none, takes nothing.
|
||||
func (r *compositionRuns) expired(boundKey string) {
|
||||
key, bound, setting := "", time.Duration(0), ""
|
||||
switch {
|
||||
case strings.HasPrefix(boundKey, runBoundPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
|
||||
case strings.HasPrefix(boundKey, runUntakenPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
|
||||
default:
|
||||
return
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
return
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
|
||||
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
|
||||
// since, ends nothing.
|
||||
other := runUntakenPrefix
|
||||
if setting == "run-untaken-bound" {
|
||||
other = runBoundPrefix
|
||||
}
|
||||
_, _, armed, err := r.boundOf(boundKey)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
|
||||
return
|
||||
}
|
||||
held, _, standing, err := r.boundOf(other + key)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
|
||||
return
|
||||
}
|
||||
if armed || (standing && held == run.Ask) {
|
||||
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
|
||||
return
|
||||
}
|
||||
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
if setting == "run-untaken-bound" {
|
||||
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
}
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
|
||||
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
|
||||
r.logf("the composition run %s past its bound is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
|
||||
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
|
||||
}
|
||||
|
||||
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
|
||||
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
|
||||
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
|
||||
}
|
||||
|
||||
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
|
||||
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
|
||||
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
|
||||
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
|
||||
if moved := r.moved.Load(); moved != nil && move {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
|
||||
(*moved)(run)
|
||||
}
|
||||
}
|
||||
ended, err := r.end(key, ask, outcome, said, verdict)
|
||||
if err != nil || !ended {
|
||||
return ended, err
|
||||
}
|
||||
if untaken {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok {
|
||||
r.sayWaits(run, run.Readers, "superseded")
|
||||
}
|
||||
}
|
||||
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
|
||||
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
|
||||
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
|
||||
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []openReader
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range run.Readers {
|
||||
if c.Ended != nil && !c.Rerun {
|
||||
continue // it ended passed already: only a reader marked rerun says its end again
|
||||
}
|
||||
if !seen[c.ID] {
|
||||
seen[c.ID] = true
|
||||
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
|
||||
type openReader struct {
|
||||
meshwants.CompositionReader
|
||||
ask string
|
||||
}
|
||||
|
||||
// keysOf are the keys of the candidate's runs that the reader reads.
|
||||
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
prefix := keysPart(candidate) + "."
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out) // by machine, as a step names its machines
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
|
||||
// handed to the runs; one that cannot be written is given again by the bus.
|
||||
func hearRunBounds(hub interface {
|
||||
Handle(string, func(jetstream.Msg)) error
|
||||
}, r *compositionRuns, logf func(string, ...any)) error {
|
||||
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardAskTaken(e), logf)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.StepProgress
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardProgress(e), logf)
|
||||
})
|
||||
}
|
||||
|
||||
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
|
||||
if err != nil {
|
||||
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
|
||||
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
|
||||
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil {
|
||||
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
|
||||
return
|
||||
}
|
||||
for _, reader := range readers {
|
||||
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
|
||||
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = reader.ask
|
||||
}
|
||||
how := c.howOf(reader.Step, true)
|
||||
if how == nil {
|
||||
c.logf("the step %s of %s is not taken up again: it is no composition or validate step (%q)", reader.ID, reader.Module, reader.Step)
|
||||
continue
|
||||
}
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), how, again)
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
|
||||
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
|
||||
|
||||
// composeCandidateKey is the seat verb's call key, asked of one node.
|
||||
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
|
||||
|
||||
// candidateKeyOf is the key the step's pipeline names its candidate by (A-226): a run on request's `run-<id>`, or for a branch
|
||||
// pipeline the branch's catalog graph revision `<branch>.<revision>`, the branch escaped as a key part as the pipeline key does.
|
||||
func candidateKeyOf(run meshwants.StepRun) (string, error) {
|
||||
switch {
|
||||
case run.Run != "":
|
||||
return nec.RunKey(run.Run)
|
||||
case run.Branch == "" || run.Revision <= 0:
|
||||
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
|
||||
}
|
||||
return nec.RevisionKey(run.Branch, run.Revision) // built by the seat's client, never by hand (A-226b)
|
||||
}
|
||||
|
||||
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
|
||||
// naming none is the trunk's, trunk the first watched branch); a branch that is not watched takes the nodes placing the module on the trunk. In name order.
|
||||
func machinesFor(placed map[string]map[string]string, module, branch, trunk string, watched bool) []string {
|
||||
if branch == "" || !watched {
|
||||
branch = trunk
|
||||
}
|
||||
var out []string
|
||||
for node, b := range placed[module] {
|
||||
if b == "" {
|
||||
b = trunk
|
||||
}
|
||||
if b == branch {
|
||||
out = append(out, node)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// machinesOf are the machines of a composition step in the world's assignments.
|
||||
func (w *world) machinesOf(module, branch string) []string {
|
||||
return machinesFor(w.placedBranches(), module, branch, w.trunkName(), w.holdsBranch(branch))
|
||||
}
|
||||
|
||||
// nodeInput is one machine's input: its bytes, or why the node gave none, or that it is away.
|
||||
type nodeInput struct {
|
||||
Node string
|
||||
Body []byte
|
||||
// Why is set when the node gave no input: its words, or the call's error.
|
||||
Why string
|
||||
// Away is a machine the watchman said away: it is not asked (§64).
|
||||
Away bool
|
||||
}
|
||||
|
||||
// compositionInputs asks the machines of a step for their inputs.
|
||||
type compositionInputs struct {
|
||||
machines func(module, branch string) []string
|
||||
// callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to
|
||||
// one node as `seat:<seat>.<verb>@<node>`, carrying the ask's id and stopping at its bound, and answers the verb's answer with
|
||||
// the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the
|
||||
// body's `call`.
|
||||
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
// bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775).
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no
|
||||
// input, naming every call.
|
||||
const (
|
||||
composeAskBoundSetting = "compose-ask-bound"
|
||||
defaultComposeAskBound = 10 * time.Second
|
||||
)
|
||||
|
||||
// composeAskBound reads the setting compose-ask-bound (§781).
|
||||
func composeAskBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per
|
||||
// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id.
|
||||
func composeAskID(stepAsk, node string) string { return stepAsk + "." + node }
|
||||
|
||||
// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step
|
||||
// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until
|
||||
// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set.
|
||||
func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput {
|
||||
bound := c.bound
|
||||
if bound <= 0 {
|
||||
bound = defaultComposeAskBound
|
||||
}
|
||||
in := nodeInput{Node: node}
|
||||
type reply struct {
|
||||
raw json.RawMessage
|
||||
err error
|
||||
}
|
||||
got := make(chan reply, 1)
|
||||
go func() {
|
||||
// The input is any size, so the verb sends it in chunks on the call token chosen here, and the file is the input (A-237b, A-237c).
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
got <- reply{nil, err}
|
||||
return
|
||||
}
|
||||
raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node,
|
||||
ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])})
|
||||
got <- reply{raw, err}
|
||||
}()
|
||||
var raw json.RawMessage
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
in.Why = "its step's wait ended: " + context.Cause(ctx).Error()
|
||||
return in
|
||||
case r := <-got:
|
||||
if r.err != nil {
|
||||
in.Why = r.err.Error()
|
||||
return in
|
||||
}
|
||||
raw = r.raw
|
||||
}
|
||||
var a ne.ComposeCandidateAnswer
|
||||
// its numbers as written (json.Number), so the input kept is the engine's own bytes' values
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&a); err != nil {
|
||||
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
|
||||
return in
|
||||
}
|
||||
switch {
|
||||
case a.Verdict == nec.Unready:
|
||||
in.Why = "unready: " + a.Refused
|
||||
return in
|
||||
case a.Chunks == 0 && len(a.Input) == 0:
|
||||
in.Why = "the engine answered no composition input"
|
||||
if a.Refused != "" {
|
||||
in.Why += ": " + a.Refused
|
||||
}
|
||||
return in
|
||||
}
|
||||
var body []byte
|
||||
if a.Chunks > 0 {
|
||||
// The composition input is the file the chunks carried, the engine's own bytes (numbers as written).
|
||||
var whole struct {
|
||||
File string `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &whole); err != nil {
|
||||
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
|
||||
return in
|
||||
}
|
||||
file, err := base64.StdEncoding.DecodeString(whole.File)
|
||||
if err != nil || len(file) == 0 {
|
||||
in.Why = fmt.Sprintf("the engine named %d chunks of composition input and the file received is empty or not base64: %v", a.Chunks, err)
|
||||
return in
|
||||
}
|
||||
body = file
|
||||
} else {
|
||||
// An answer with the input inline (an engine before A-237b, read until the checker is removed, A-237c, G-6 §321): the object, as it was.
|
||||
var err error
|
||||
if body, err = json.Marshal(a.Input); err != nil {
|
||||
in.Why = "its composition input cannot be encoded again: " + err.Error()
|
||||
return in
|
||||
}
|
||||
}
|
||||
in.Body = body
|
||||
return in
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G A-226: the candidate a step asks is `run-<id>` for a run on request, else `<branch>.<revision>`.
|
||||
func TestTheCandidateKeyOfAStep(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
run meshwants.StepRun
|
||||
want string
|
||||
fail bool
|
||||
}{
|
||||
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
|
||||
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 362}, want: "main.362"},
|
||||
{run: meshwants.StepRun{Branch: "main"}, fail: true},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 100000000000}, fail: true}, // a number the client's key would read as a heads hash
|
||||
{run: meshwants.StepRun{Run: "a.b"}, fail: true}, // a run id the client refuses
|
||||
} {
|
||||
got, err := candidateKeyOf(c.run)
|
||||
if (err != nil) != c.fail || got != c.want {
|
||||
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
|
||||
}
|
||||
// A-226b: the node-engine reads every key the step names with the client's one parser.
|
||||
if !c.fail {
|
||||
if _, err := nec.ParseCandidateKey(got); err != nil {
|
||||
t.Errorf("%+v: the key %q is refused by the seat's parser: %v", c.run, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A-252: the machines of a step are the nodes whose placement names its branch; none named is the trunk; an unwatched branch takes
|
||||
// the trunk's nodes.
|
||||
func TestTheMachinesOfAStep(t *testing.T) {
|
||||
placed := map[string]map[string]string{"m": {"ace": "", "g14": "feat", "novox": "main", "shanks": "feat"}, "other": {"ace": ""}}
|
||||
for _, c := range []struct {
|
||||
branch string
|
||||
watched bool
|
||||
want string
|
||||
}{
|
||||
{"main", true, "ace novox"},
|
||||
{"feat", true, "g14 shanks"},
|
||||
{"gone", false, "ace novox"},
|
||||
{"", true, "ace novox"},
|
||||
} {
|
||||
if got := strings.Join(machinesFor(placed, "m", c.branch, trunkBranch, c.watched), " "); got != c.want {
|
||||
t.Errorf("branch %q watched %v: %q, want %q", c.branch, c.watched, got, c.want)
|
||||
}
|
||||
}
|
||||
if got := machinesFor(placed, "none", "main", trunkBranch, true); len(got) != 0 {
|
||||
t.Errorf("a module placed nowhere has machines %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
|
||||
// silent or unready node says why, and a machine away is not asked.
|
||||
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
// the machines are asked side by side, so the record of what each was asked is guarded.
|
||||
var mu sync.Mutex
|
||||
asked := map[string]string{}
|
||||
asks := map[string]process.KeptAsk{}
|
||||
call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
mu.Lock()
|
||||
asked[node], asks[node] = key+" "+string(raw), ask
|
||||
mu.Unlock()
|
||||
switch node {
|
||||
case "g14":
|
||||
return nil, errors.New("no responders")
|
||||
case "novox":
|
||||
return json.RawMessage(`{"candidate":"run-r1","node":"novox","verdict":"unready","refused":"starting","modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
if err != nil || len(got) != 4 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
// With a call token of 128 random bits (32 hex digits), the verb being marked chunks.
|
||||
if k, body, _ := strings.Cut(asked["ace"], " "); k != "seat:node-engine.compose-candidate" ||
|
||||
!regexp.MustCompile(`^\{"call":"[0-9a-f]{32}","candidate":"run-r1","input":true\}$`).MatchString(body) {
|
||||
t.Errorf("asked %q", asked["ace"])
|
||||
}
|
||||
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
|
||||
t.Errorf("ace: %+v", got[0])
|
||||
}
|
||||
// Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound.
|
||||
if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" {
|
||||
t.Errorf("the kept asks %+v", asks)
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
|
||||
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
|
||||
}
|
||||
if _, was := asked["shanks"]; was || !got[3].Away {
|
||||
t.Errorf("the machine away is asked or not said away: %+v", got[3])
|
||||
}
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
|
||||
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
t.Error("a step with no machine is gathered")
|
||||
}
|
||||
}
|
||||
|
||||
// A-237b: the composition input is any size, so a node names the chunks it sent and the assigner reads the file the chunked call
|
||||
// received as the input, its numbers as written; a node that names chunks and sends no file is said so.
|
||||
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
|
||||
file := base64.StdEncoding.EncodeToString([]byte(input))
|
||||
call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@")
|
||||
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
|
||||
if node == "g14" {
|
||||
return json.RawMessage(answer + `}`), nil // names chunks, none received
|
||||
}
|
||||
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if got[0].Why != "" || string(got[0].Body) != input {
|
||||
t.Errorf("ace: why %q, %d bytes, want the file received (%d bytes) as the input", got[0].Why, len(got[0].Body), len(input))
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "chunks") || got[1].Body != nil {
|
||||
t.Errorf("g14: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
|
||||
// is not asked; no machine is an error.
|
||||
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
|
||||
key, err := candidateKeyOf(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := c.machines(run.Module, run.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return nil, errors.New("no machine")
|
||||
}
|
||||
out := make([]nodeInput, len(nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
if away != nil && away(node) {
|
||||
out[i] = nodeInput{Node: node, Away: true}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
|
||||
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
|
||||
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
|
||||
// could not run.
|
||||
|
||||
const (
|
||||
// readsBound is how long a read is made again before the step could not run.
|
||||
readsBound = 3 * time.Minute
|
||||
// readsFirstWait and readsMaxWait bound the wait between two tries.
|
||||
readsFirstWait = time.Second
|
||||
readsMaxWait = 15 * time.Second
|
||||
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
|
||||
readsKept = 8
|
||||
)
|
||||
|
||||
// readKey names one revision of one watched branch.
|
||||
type readKey struct {
|
||||
branch string
|
||||
revision int
|
||||
}
|
||||
|
||||
// readSlot is one revision's read: done closes when half or err is set.
|
||||
type readSlot struct {
|
||||
done chan struct{}
|
||||
half mcc.Half
|
||||
err error
|
||||
}
|
||||
|
||||
// compositionReads shares the reads of composition steps.
|
||||
type compositionReads struct {
|
||||
mu sync.Mutex
|
||||
revs map[readKey]*readSlot
|
||||
order []readKey
|
||||
seats map[string]json.RawMessage // by seat and digest: the definition, raw
|
||||
}
|
||||
|
||||
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
|
||||
func transient(err error, said string) bool {
|
||||
text := said
|
||||
if err != nil {
|
||||
text = err.Error() + " " + said
|
||||
}
|
||||
text = strings.ToLower(text)
|
||||
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
|
||||
if strings.Contains(text, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
|
||||
// a good answer, or the last fault.
|
||||
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
|
||||
start := r.now()
|
||||
wait := readsFirstWait
|
||||
for try := 1; ; try++ {
|
||||
raw, err := r.call(key, body)
|
||||
var inner json.RawMessage
|
||||
var said string
|
||||
if err == nil {
|
||||
var ok bool
|
||||
if inner, said, ok = buscalls.Unwrap(raw); ok {
|
||||
return inner, "", nil
|
||||
}
|
||||
}
|
||||
if !transient(err, said) {
|
||||
return nil, said, err
|
||||
}
|
||||
if r.now().Sub(start)+wait > readsBound {
|
||||
return nil, said, err
|
||||
}
|
||||
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
|
||||
if serr := r.waitFor(ctx, wait); serr != nil {
|
||||
return nil, said, serr
|
||||
}
|
||||
if wait *= 2; wait > readsMaxWait {
|
||||
wait = readsMaxWait
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor waits d, or until ctx ends.
|
||||
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
|
||||
if r.sleep != nil {
|
||||
return r.sleep(ctx, d)
|
||||
}
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
|
||||
// failed is not kept, so the next step reads again.
|
||||
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
|
||||
c := r.reads
|
||||
c.mu.Lock()
|
||||
if c.revs == nil {
|
||||
c.revs = map[readKey]*readSlot{}
|
||||
}
|
||||
if s, ok := c.revs[key]; ok {
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-s.done:
|
||||
if s.err == nil {
|
||||
return s.half, nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return mcc.Half{}, ctx.Err()
|
||||
}
|
||||
// The read this step waited for failed: read it here (the failed slot is already dropped).
|
||||
return r.half(ctx, key, read)
|
||||
}
|
||||
s := &readSlot{done: make(chan struct{})}
|
||||
c.revs[key] = s
|
||||
c.order = append(c.order, key)
|
||||
for len(c.order) > readsKept {
|
||||
delete(c.revs, c.order[0])
|
||||
c.order = c.order[1:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
s.half, s.err = read()
|
||||
if s.err != nil {
|
||||
c.mu.Lock()
|
||||
if c.revs[key] == s {
|
||||
delete(c.revs, key)
|
||||
}
|
||||
for i, k := range c.order {
|
||||
if k == key {
|
||||
c.order = append(c.order[:i], c.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
close(s.done)
|
||||
return s.half, s.err
|
||||
}
|
||||
|
||||
// seatKept is the definition kept of a seat at a digest.
|
||||
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
d, ok := c.seats[seat+"@"+digest]
|
||||
return d, ok
|
||||
}
|
||||
|
||||
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.seats == nil {
|
||||
c.seats = map[string]json.RawMessage{}
|
||||
}
|
||||
c.seats[seat+"@"+digest] = def
|
||||
}
|
||||
|
||||
func unreadErr(what string, err error, said string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", what, said)
|
||||
}
|
||||
@@ -1,869 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
|
||||
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
|
||||
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests
|
||||
// and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
|
||||
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
|
||||
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
|
||||
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
|
||||
// watchman said away (machine-away) is not composed and passes with a warning (§64).
|
||||
|
||||
const (
|
||||
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
|
||||
// asks are put on and the first of their step keys (§467, §481, §600, A-163).
|
||||
runAsker = meshwants.Seat
|
||||
// changeCase is the one case each request composes.
|
||||
changeCase = "change"
|
||||
// inputVerbKey is the verb a run ask's input is served on: the seat mesh-wants' input (§354).
|
||||
inputVerbKey = "seat:" + meshwants.Seat + "." + meshwants.VerbInput
|
||||
// entryVerbKey and seatDefinitionsKey are the mesh-catalog seat verbs the step reads.
|
||||
entryVerbKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbEntry
|
||||
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
|
||||
)
|
||||
|
||||
// compositionRun runs a composition step.
|
||||
type compositionRun struct {
|
||||
// runs are the composition runs of the state composition-runs (compositionruns.go).
|
||||
runs *compositionRuns
|
||||
inputs *compositionInputs
|
||||
keeper *inputKeeper
|
||||
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
|
||||
call func(key string, body any) (json.RawMessage, error)
|
||||
// callFile asks a seat verb held once for the mesh whose answer comes in chunks, as a kept call (process.Conn.CallFileKept),
|
||||
// answering the verb's answer with the bytes received as its member `file`, base64: the validate step's seat-list (H-A-443).
|
||||
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
// trunk is the trunk's catalog half as the assigner holds it.
|
||||
trunk func() (mcc.Half, bool)
|
||||
// newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's
|
||||
// watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across
|
||||
// them (design 47 H-A-143, G §532). Nil: the step's revision alone.
|
||||
newest func() map[string]mcc.Half
|
||||
watched func(module string) []string
|
||||
// assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists;
|
||||
// assignment is the machine's assignments whole, which the case composes with those modules left out (H-A-148).
|
||||
assigned func(node string) []string
|
||||
assignment func(node string) (mma.Assignments, bool)
|
||||
// findings is where the step raises the finding assignment-unlisted of each assignment it leaves out (H-A-149); the pass over
|
||||
// the assignments clears it.
|
||||
findings assign.Findings
|
||||
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
|
||||
running func(node, module string) (ne.Running, bool)
|
||||
// away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow),
|
||||
// so a durable on the watchman's streams has one hearer.
|
||||
away func(machine string) bool
|
||||
// put puts a run ask on the build seat's check queue under the assigner's name.
|
||||
put func(ctx context.Context, a buildstep.StepAsk) error
|
||||
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
|
||||
reads *compositionReads
|
||||
sleep func(ctx context.Context, d time.Duration) error
|
||||
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
|
||||
ask buildstep.AskVerb
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// machineEnd is what one machine came to.
|
||||
type machineEnd struct {
|
||||
failed, couldNot string
|
||||
warnings []string
|
||||
// judged is set where the composer judged the machine, composing or not (A-467a).
|
||||
judged bool
|
||||
}
|
||||
|
||||
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
|
||||
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
|
||||
// compose, could not run when a machine could not be composed or was not judged, nothing to run when no node places the module on
|
||||
// the branch (H-A-179), else passed with a warning per machine away.
|
||||
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
nodes := r.inputs.machines(step.Module, step.Branch)
|
||||
if len(nodes) == 0 { // a mover no machine holds on the branch has nothing to judge: nothing to run, passed (H-A-179, ADR 0682 §8)
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeNothingToRun,
|
||||
Said: fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)}
|
||||
}
|
||||
reader := readerOf(id, step)
|
||||
prep := &runPrep{r: r, step: step}
|
||||
var ends []machineEnd
|
||||
var keys []string
|
||||
for _, node := range nodes {
|
||||
if r.away != nil && r.away(node) {
|
||||
ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}})
|
||||
continue
|
||||
}
|
||||
join := r.runs.join
|
||||
if step.Rerun { // a step asked again (§727)
|
||||
join = r.runs.rejoin
|
||||
}
|
||||
j, err := join(reader, candidate, node, buildstep.NewID())
|
||||
if err != nil {
|
||||
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
|
||||
continue
|
||||
}
|
||||
if j.started {
|
||||
go r.start(ctx, j.key, j.run, prep, id)
|
||||
}
|
||||
keys = append(keys, j.key)
|
||||
}
|
||||
return r.read(ctx, step.Module, keys, ends)
|
||||
}
|
||||
|
||||
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
|
||||
// runs it reads and their asks (§723, §729).
|
||||
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
|
||||
setShared(ctx, r.runs.sharedOf(keys, nil))
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
runs := r.runs.await(ctx, keys)
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
|
||||
for _, key := range keys {
|
||||
ends = append(ends, judgeRun(module, runs[key]))
|
||||
}
|
||||
out := outcomeOf(ends)
|
||||
out.Shared = r.runs.sharedOf(keys, runs)
|
||||
return out
|
||||
}
|
||||
|
||||
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
|
||||
func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
keys, err := r.runs.keysOf(candidate, id)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's composition runs are not read: " + err.Error())
|
||||
}
|
||||
return r.read(ctx, step.Module, keys, nil)
|
||||
}
|
||||
|
||||
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
|
||||
// machine never passes (A-467a), each machine not judged named with why.
|
||||
func outcomeOf(ends []machineEnd) stepOutcome {
|
||||
var out stepOutcome
|
||||
var failed, couldNot []string
|
||||
judged := 0
|
||||
for _, e := range ends {
|
||||
if e.judged {
|
||||
judged++
|
||||
}
|
||||
failed = appendSome(failed, e.failed)
|
||||
couldNot = appendSome(couldNot, e.couldNot)
|
||||
out.Warnings = append(out.Warnings, e.warnings...)
|
||||
}
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
|
||||
case len(couldNot) > 0:
|
||||
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
|
||||
case judged == 0:
|
||||
said := "no machine was judged"
|
||||
if len(out.Warnings) == 0 {
|
||||
said += ": the step has no machine"
|
||||
} else {
|
||||
said += ": " + strings.Join(out.Warnings, "; ")
|
||||
}
|
||||
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, said
|
||||
default:
|
||||
out.Outcome = meshwants.StepEndedOutcomePassed
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendSome(to []string, s string) []string {
|
||||
if s == "" {
|
||||
return to
|
||||
}
|
||||
return append(to, s)
|
||||
}
|
||||
|
||||
func couldNotRun(said string) stepOutcome {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
|
||||
}
|
||||
|
||||
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
|
||||
// case's manifests, and the composer program per machine. The case carries no verdicts: the step validate judged the
|
||||
// manifests before the composition step (design 47 H-A-112, H-A-108).
|
||||
type runPrep struct {
|
||||
r *compositionRun
|
||||
step meshwants.StepRun
|
||||
once sync.Once
|
||||
// read once
|
||||
seats []json.RawMessage
|
||||
manifests map[string]composer.Manifest
|
||||
none map[string]string
|
||||
programOf func(node string) (buildstep.RunProgram, string, map[string]string, error)
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *runPrep) get(ctx context.Context) error {
|
||||
p.once.Do(func() {
|
||||
half, err := p.r.revision(ctx, p.step)
|
||||
if err != nil {
|
||||
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if p.r.newest != nil && p.r.watched != nil {
|
||||
half = acrossBranches(p.step, half, p.r.newest(), p.r.watched)
|
||||
}
|
||||
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
|
||||
return
|
||||
}
|
||||
if p.manifests, p.none, p.err = caseOf(half); p.err != nil {
|
||||
return
|
||||
}
|
||||
p.programOf, p.err = p.r.programs(ctx, half)
|
||||
})
|
||||
return p.err
|
||||
}
|
||||
|
||||
// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`,
|
||||
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
|
||||
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) {
|
||||
node, ask := run.Machine, run.Ask
|
||||
end := func(said string) {
|
||||
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
|
||||
r.logf("the composition run %s is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
// The one set of unlisted assignments, read once at the run's start and carried on the run to the leaving-out, the finding and
|
||||
// every step's judgement, the resume's too (design 47 H-A-152, H-A-153).
|
||||
unlisted := r.unlistedOn(node)
|
||||
if len(unlisted) > 0 {
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Unlisted = unlisted
|
||||
return true
|
||||
}); err != nil {
|
||||
end(node + ": the run's unlisted assignments are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := prep.get(ctx); err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405)
|
||||
if in.Why != "" {
|
||||
end(node + " gave no composition input: " + in.Why)
|
||||
return
|
||||
}
|
||||
program, engineFp, pins, err := prep.programOf(node)
|
||||
if err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
cs := composer.Case{Name: changeCase, Manifests: prep.manifests, None: prep.none}
|
||||
if left, err := r.leftOut(node, unlisted); err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
} else if left != nil {
|
||||
cs.Assignments = left
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
|
||||
Cases: []composer.Case{cs}, Seats: prep.seats,
|
||||
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
end(node + ": its request cannot be encoded: " + err.Error())
|
||||
return
|
||||
}
|
||||
digest, err := r.keeper.keep(ask, raw)
|
||||
if err != nil {
|
||||
r.logf("the composition request of %s is refused: %v", node, err)
|
||||
end(node + ": its request is not kept: " + err.Error())
|
||||
return
|
||||
}
|
||||
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
|
||||
if err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
|
||||
c.Input = digest
|
||||
return true
|
||||
}); err != nil {
|
||||
end(node + ": the run's program and input are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit},
|
||||
Packages: pins} // the composer is built against its entry's pins, as every build (H-A-203, H-A-192)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
end(node + ": its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's
|
||||
// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b).
|
||||
const notTried = "not tried: the search reached its bound"
|
||||
|
||||
// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else
|
||||
// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is
|
||||
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
|
||||
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
|
||||
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
|
||||
//
|
||||
// The run's unlisted are the modules the machine is assigned that no watched branch's newest revision lists, left out of its
|
||||
// composition, as the run's start read them (H-A-152, H-A-153): each is said on every path, in the machine's failed or
|
||||
// could-not-run words where one is set, else as a warning, never the machine's failure and never dropped in silence (H-A-143,
|
||||
// H-A-144, H-A-148).
|
||||
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
|
||||
end := judgeVerdict(module, run)
|
||||
for _, u := range run.Unlisted {
|
||||
said := run.Machine + " is assigned " + u + ", which no watched branch lists (H-A-143)"
|
||||
switch {
|
||||
case end.failed != "":
|
||||
end.failed += "; " + said
|
||||
case end.couldNot != "":
|
||||
end.couldNot += "; " + said
|
||||
default:
|
||||
end.warnings = append(end.warnings, said)
|
||||
}
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
// judgeVerdict is judgeRun before the unlisted modules are said.
|
||||
func judgeVerdict(module string, run meshwants.CompositionRun) machineEnd {
|
||||
node := run.Machine
|
||||
if run.Ended == nil {
|
||||
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
|
||||
}
|
||||
if run.Outcome != meshwants.CompositionRunOutcomePassed {
|
||||
return machineEnd{couldNot: run.Said}
|
||||
}
|
||||
m, err := machineOf(run.Verdict)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
|
||||
}
|
||||
if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467)
|
||||
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
|
||||
}
|
||||
why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")
|
||||
switch {
|
||||
case m.Blamed == nil && !m.Composes: // an old composer (§721)
|
||||
return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}}
|
||||
case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452)
|
||||
return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why}
|
||||
}
|
||||
end := machineEnd{judged: true}
|
||||
var own, others []string
|
||||
for _, b := range m.Blamed {
|
||||
switch {
|
||||
case b.Module == module && b.Problem == notTried:
|
||||
end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module)
|
||||
case b.Module == module:
|
||||
own = append(own, b.Problem)
|
||||
default:
|
||||
others = append(others, node+" holds "+b.Module+": "+b.Problem)
|
||||
}
|
||||
}
|
||||
if len(own) > 0 {
|
||||
end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), ""
|
||||
}
|
||||
end.warnings = others
|
||||
return end
|
||||
}
|
||||
|
||||
// machineOf is the composer's verdict of one machine as a run keeps it.
|
||||
func machineOf(v map[string]any) (composer.Machine, error) {
|
||||
var m composer.Machine
|
||||
raw, err := json.Marshal(v)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &m)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
|
||||
func runVerdict(node string, answer []byte) (map[string]any, string) {
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(answer, &v); err != nil {
|
||||
return nil, node + ": the composer answered no verdict: " + err.Error()
|
||||
}
|
||||
if v.V != composer.Version {
|
||||
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name != changeCase {
|
||||
continue
|
||||
}
|
||||
m, ok := c.Machines[node]
|
||||
if !ok {
|
||||
return nil, node + ": the composer's verdict names no such machine"
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
var out map[string]any
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &out)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
return nil, node + ": the composer's verdict has no case " + changeCase
|
||||
}
|
||||
|
||||
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
|
||||
// it (compositionreads.go).
|
||||
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
|
||||
if run.Revision <= 0 {
|
||||
return mcc.Half{}, errors.New("the step names no catalog revision")
|
||||
}
|
||||
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
|
||||
}
|
||||
|
||||
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
|
||||
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
|
||||
for {
|
||||
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
|
||||
if err != nil {
|
||||
return mcc.Half{}, err
|
||||
}
|
||||
if body == nil {
|
||||
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
|
||||
}
|
||||
var a meshcatalog.RevisionAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return mcc.Half{}, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
|
||||
}
|
||||
if a.NoneYet || a.Revision == nil || a.Revision.Revision != run.Revision {
|
||||
return mcc.Half{}, fmt.Errorf("%s answered no revision %d of %q", mcc.RevisionVerb, run.Revision, run.Branch)
|
||||
}
|
||||
if in.After == "" {
|
||||
half.Revision = *a.Revision // the first page names what the revision was read from, its definitions included
|
||||
}
|
||||
for _, e := range a.Entries {
|
||||
half.Entries[e.Module] = e
|
||||
}
|
||||
if a.Next == "" {
|
||||
return half, nil
|
||||
}
|
||||
in.After = a.Next
|
||||
}
|
||||
}
|
||||
|
||||
// seats are the definitions the composer is handed, raw. A revision that names definitions (A-261): the definition of each (seat, digest)
|
||||
// pair, from mesh-catalog's seat-definitions where the trunk's digest is the revision's; a digest neither holds is an error naming the seat
|
||||
// and the digest. A revision that names none (A-234's stand-in, A-467): every definition of the trunk seat list, read through
|
||||
// seat-definitions; a list or a definition that cannot be read is an error naming it, never an empty list.
|
||||
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
|
||||
if len(defs) == 0 {
|
||||
return r.trunkSeats(ctx)
|
||||
}
|
||||
out := make([]json.RawMessage, 0, len(defs))
|
||||
for _, d := range defs {
|
||||
def, err := r.seatAt(ctx, d.Seat, d.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, def)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trunkSeats are the definitions of every seat of the trunk's seat list (seat-definitions with no seat names them with their digests).
|
||||
func (r *compositionRun) trunkSeats(ctx context.Context) ([]json.RawMessage, error) {
|
||||
what := "the trunk's seat list is not read"
|
||||
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{})
|
||||
if err != nil || body == nil {
|
||||
return nil, unreadErr(what, err, said)
|
||||
}
|
||||
var list meshcatalog.SeatDefinitionsAnswer
|
||||
if err := json.Unmarshal(body, &list); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
if len(list.Seats) == 0 {
|
||||
return nil, errors.New(what + ": mesh-catalog's seat-definitions names no seat, so the composer has no seat list to judge by")
|
||||
}
|
||||
out := make([]json.RawMessage, 0, len(list.Seats))
|
||||
for _, s := range list.Seats {
|
||||
def, err := r.seatAt(ctx, s.Seat, s.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, def)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seatAt is the definition of a seat at a digest, kept once read (a definition at a digest never changes).
|
||||
func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json.RawMessage, error) {
|
||||
if def, ok := r.reads.seatKept(seat, digest); ok {
|
||||
return def, nil
|
||||
}
|
||||
what := "the definition of the seat " + seat + " is not read"
|
||||
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: seat})
|
||||
if err != nil || body == nil {
|
||||
return nil, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.SeatDefinitionsAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
if a.Entry == nil || a.Entry.Digest != digest || a.Entry.Definition == nil {
|
||||
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", seat, digest)
|
||||
}
|
||||
def, err := json.Marshal(a.Entry.Definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.reads.keepSeat(seat, digest, def)
|
||||
return def, nil
|
||||
}
|
||||
|
||||
// caseOf is the case's manifests and none from the revision's entries: each manifest whole, and the revision's none words of each
|
||||
// module it gives no fingerprint, whose build can never be made (design 47 G A-470j, A-470k). It sets no verdicts (H-A-112).
|
||||
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, error) {
|
||||
manifests := map[string]composer.Manifest{}
|
||||
var none map[string]string
|
||||
for name, e := range half.Entries {
|
||||
if len(e.Manifest) == 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
|
||||
}
|
||||
manifests[name] = composer.Manifest{Raw: raw}
|
||||
if e.Fp == "" && e.None != "" {
|
||||
if none == nil {
|
||||
none = map[string]string{}
|
||||
}
|
||||
none[name] = e.None
|
||||
}
|
||||
}
|
||||
return manifests, none, nil
|
||||
}
|
||||
|
||||
// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, H-A-144, G §532): each module's entry from
|
||||
// the newest revision of the first of its watched branches, in its registration's order, that lists it, none for a module with no
|
||||
// watched branches; the candidate's revision replaces its
|
||||
// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest
|
||||
// or the branch's is not held. The half keeps the candidate's revision, its definitions with it.
|
||||
func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half {
|
||||
halves := make(map[string]mcc.Half, len(newest)+1)
|
||||
for b, h := range newest {
|
||||
halves[b] = h
|
||||
}
|
||||
if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision {
|
||||
halves[step.Branch] = candidate
|
||||
} else {
|
||||
mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)}
|
||||
moved := []string{step.Module}
|
||||
for _, m := range candidate.Revision.Movers {
|
||||
moved = append(moved, m.Module)
|
||||
}
|
||||
for _, m := range moved {
|
||||
if e, ok := candidate.Entries[m]; ok {
|
||||
mix.Entries[m] = e
|
||||
} else {
|
||||
delete(mix.Entries, m)
|
||||
}
|
||||
}
|
||||
halves[step.Branch] = mix
|
||||
}
|
||||
out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, h := range halves {
|
||||
for name := range h.Entries {
|
||||
if _, done := out.Entries[name]; done {
|
||||
continue
|
||||
}
|
||||
for _, b := range watched(name) { // none: no watched branch lists it, it is unlisted (H-A-144)
|
||||
if e, ok := halves[b].Entries[name]; ok {
|
||||
out.Entries[name] = e
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unlistedOn is the one set of unlisted assignments of node (H-A-152): the modules the assignments place there that the newest
|
||||
// revision of none of their watched branches lists, a module with no watched branches among them (H-A-143, H-A-144), by name.
|
||||
// The candidate never enters it, so a module the change drops while still assigned stays in the composition and fails it as the
|
||||
// change's own fault (ADR 0578 §2). Where the newest revisions are not followed yet nothing is unlisted: none while no branch's
|
||||
// newest is held, and not a module one of whose watched branches' newest is not held yet.
|
||||
func (r *compositionRun) unlistedOn(node string) []string {
|
||||
if r.assigned == nil || r.newest == nil || r.watched == nil {
|
||||
return nil
|
||||
}
|
||||
newest := r.newest()
|
||||
if len(newest) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range r.assigned(node) {
|
||||
watched := r.watched(m)
|
||||
if !listedOn(newest, watched, m) && !slices.ContainsFunc(watched, func(b string) bool { _, held := newest[b]; return !held }) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
// leftOut is the case's assignments of node with each module of the run's unlisted set left out, or nil where none is (the case
|
||||
// then composes the input's): a stale assignment is a fault of the assignments, not of the change judged, so it is left out of the
|
||||
// composition, said in the journal, raised as assignment-unlisted and, at the judgement, said as unlisted from the same set
|
||||
// (design 47 H-A-148, H-A-149, H-A-152, H-A-143; ADR 0578 §2).
|
||||
func (r *compositionRun) leftOut(node string, set []string) (map[string]json.RawMessage, error) {
|
||||
if r.assignment == nil || len(set) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
a, ok := r.assignment(node)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
var unlisted []string
|
||||
for _, m := range a.Modules {
|
||||
if slices.Contains(set, m.Module) {
|
||||
unlisted = append(unlisted, m.Module)
|
||||
}
|
||||
}
|
||||
if len(unlisted) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
a.Modules = slices.DeleteFunc(slices.Clone(a.Modules), func(m mma.PlacedModule) bool { return slices.Contains(unlisted, m.Module) })
|
||||
raw, err := mmac.EncodeAssignments(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its assignments with %s left out cannot be encoded: %w", strings.Join(unlisted, ", "), err)
|
||||
}
|
||||
r.logf("%s: the composition leaves out %s, assigned there and listed on no watched branch (H-A-148)", node, strings.Join(unlisted, ", "))
|
||||
for _, m := range unlisted {
|
||||
r.raiseUnlisted(node, m)
|
||||
}
|
||||
return map[string]json.RawMessage{node: raw}, nil
|
||||
}
|
||||
|
||||
// raiseUnlisted writes the finding assignment-unlisted of module on node (H-A-149); one not written is said in the journal.
|
||||
func (r *compositionRun) raiseUnlisted(node, module string) {
|
||||
if r.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := r.now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
f, err := assign.UnlistedFinding(node, module, now())
|
||||
if err == nil {
|
||||
err = r.findings.Put(f)
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("the finding %s is not written: %v", assign.UnlistedKey(node, module), err)
|
||||
}
|
||||
}
|
||||
|
||||
// listedOn says whether the newest revision of one of module's watched branches lists it (H-A-143, H-A-144).
|
||||
func listedOn(newest map[string]mcc.Half, watched []string, module string) bool {
|
||||
for _, b := range watched {
|
||||
if _, ok := newest[b].Entries[module]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
|
||||
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
|
||||
for _, name := range slices.Sorted(mapsKeys(entries)) {
|
||||
for _, s := range entries[name].Serves {
|
||||
// The catalog writes a served seat verb as `seat:<seat>.<verb>` (as the compose words above are made).
|
||||
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), ne.Seat+".") {
|
||||
return entries[name], true
|
||||
}
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, false
|
||||
}
|
||||
|
||||
func mapsKeys(m map[string]meshcatalog.CatalogEntry) func(func(string) bool) {
|
||||
return func(yield func(string) bool) {
|
||||
for k := range m {
|
||||
if !yield(k) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// programOfEntry is the composer built from a node-engine entry: its repository, commit and directory, the composer's package.
|
||||
func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
|
||||
dir := e.Path
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
if e.Repository == "" || e.Commit == "" {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("the node-engine's entry %s names no repository or commit", e.Fp)
|
||||
}
|
||||
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
|
||||
}
|
||||
|
||||
// pinsOfEntry are the mesh packages' versions an entry pins, the ones it names with a version (H-A-203, H-A-183): the pins
|
||||
// a run's program is built against, in the form the entry keeps and the builder sends. nil when it pins none.
|
||||
func pinsOfEntry(e meshcatalog.CatalogEntry) map[string]string {
|
||||
var out map[string]string
|
||||
for path, version := range e.Packages {
|
||||
if version == "" {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[path] = version
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
|
||||
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
|
||||
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
|
||||
// `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, map[string]string, error), error) {
|
||||
engine, ok := nodeEngineEntry(half.Entries)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
|
||||
}
|
||||
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
|
||||
p, err := programOfEntry(engine)
|
||||
return func(string) (buildstep.RunProgram, string, map[string]string, error) {
|
||||
return p, engine.Fp, pinsOfEntry(engine), err
|
||||
}, nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
type built struct {
|
||||
program buildstep.RunProgram
|
||||
pins map[string]string
|
||||
}
|
||||
byFp := map[string]built{}
|
||||
return func(node string) (buildstep.RunProgram, string, map[string]string, error) {
|
||||
running, ok := r.running(node, engine.Module)
|
||||
if !ok || running.Fingerprint == "" {
|
||||
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
|
||||
engine.Module, engine.Fp)
|
||||
p, err := programOfEntry(engine)
|
||||
return p, engine.Fp, pinsOfEntry(engine), err
|
||||
}
|
||||
fp := running.Fingerprint
|
||||
mu.Lock()
|
||||
b, ok := byFp[fp]
|
||||
mu.Unlock()
|
||||
if ok {
|
||||
return b.program, fp, b.pins, nil
|
||||
}
|
||||
what := "the entry of the node-engine's build " + fp + " is not read"
|
||||
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
|
||||
if err != nil || body == nil {
|
||||
return buildstep.RunProgram{}, fp, nil, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.EntryAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return buildstep.RunProgram{}, fp, nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
p, err := programOfEntry(a.Entry)
|
||||
if err != nil {
|
||||
return p, fp, nil, err
|
||||
}
|
||||
pins := pinsOfEntry(a.Entry)
|
||||
mu.Lock()
|
||||
byFp[fp] = built{program: p, pins: pins}
|
||||
mu.Unlock()
|
||||
return p, fp, pins, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// trunkOf is the trunk's catalog half the assigner's views hold, trunk naming the first watched branch, or false while none is
|
||||
// held.
|
||||
func trunkOf(iv *catalogViews, trunk func() string) func() (mcc.Half, bool) {
|
||||
return func() (mcc.Half, bool) {
|
||||
if iv == nil || iv.h == nil {
|
||||
return mcc.Half{}, false
|
||||
}
|
||||
h, ok := iv.h.views()[trunk()]
|
||||
return h, ok
|
||||
}
|
||||
}
|
||||
|
||||
// runAskPublisher puts a run ask on the Go build seat's check queue under the assigner's name (`accept.check.assigner`, §600), its
|
||||
// id the message id, so the stream takes it once.
|
||||
func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildstep.StepAsk) error {
|
||||
return func(ctx context.Context, a buildstep.StepAsk) error {
|
||||
raw, err := buildstep.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, a.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, raw, jetstream.WithMsgID(a.ID))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed.
|
||||
func (iv *catalogViews) newestOf() func() map[string]mcc.Half {
|
||||
if iv == nil {
|
||||
return nil
|
||||
}
|
||||
return func() map[string]mcc.Half {
|
||||
if iv.h == nil {
|
||||
return map[string]mcc.Half{}
|
||||
}
|
||||
return iv.h.views()
|
||||
}
|
||||
}
|
||||
|
||||
// listed says whether one of module's watched branches lists it in the newest revision the views hold (H-A-149's clear).
|
||||
func (iv *catalogViews) listed(module string) bool {
|
||||
if iv == nil || iv.h == nil {
|
||||
return false
|
||||
}
|
||||
return listedOn(iv.h.views(), iv.h.watchedOf(module), module)
|
||||
}
|
||||
|
||||
// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed.
|
||||
func (iv *catalogViews) watchedOf(module string) []string {
|
||||
if iv == nil || iv.h == nil {
|
||||
return nil
|
||||
}
|
||||
return iv.h.watchedOf(module)
|
||||
}
|
||||
@@ -1,807 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
const (
|
||||
commitE1 = "1111111111111111111111111111111111111111"
|
||||
commitE2 = "2222222222222222222222222222222222222222"
|
||||
commitE3 = "3333333333333333333333333333333333333333"
|
||||
)
|
||||
|
||||
// runStand is a composition run over stand-ins of the mesh-catalog seat, the machines' engines, the build seat and its step-ended.
|
||||
type runStand struct {
|
||||
mu sync.Mutex
|
||||
asks map[string]buildstep.StepAsk // by node
|
||||
requests map[string]composer.Request // by node
|
||||
entries []string // the fingerprints `entry` was asked
|
||||
r *compositionRun
|
||||
runs *compositionRuns
|
||||
puts int // the run asks put
|
||||
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
|
||||
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
|
||||
noDefinitions bool
|
||||
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
|
||||
unjudged string
|
||||
listFault error
|
||||
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
|
||||
verdict func(node string) (composer.Machine, bool)
|
||||
}
|
||||
|
||||
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
|
||||
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
|
||||
engine := func(fp, commit string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
|
||||
Manifest: map[string]any{"module": "engine"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25-" + fp, "git.novox.be/novox/x/go": ""}}
|
||||
}
|
||||
revisionCommit := commitE1
|
||||
if revisionEngine != "src2:e1" {
|
||||
revisionCommit = commitE3
|
||||
}
|
||||
pages := map[string]meshcatalog.RevisionAnswer{
|
||||
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
|
||||
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
|
||||
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
|
||||
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
|
||||
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
switch key {
|
||||
case mcc.RevisionVerb:
|
||||
var in meshcatalog.RevisionInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
if in.Revision != 7 || in.Branch != "main" {
|
||||
return nil, errors.New("asked another revision")
|
||||
}
|
||||
page := pages[in.After]
|
||||
if s.noDefinitions && page.Revision != nil {
|
||||
rev := *page.Revision
|
||||
rev.Definitions = nil
|
||||
page.Revision = &rev
|
||||
}
|
||||
return json.Marshal(page)
|
||||
case seatDefinitionsKey:
|
||||
var sin meshcatalog.SeatDefinitionsInput
|
||||
_ = json.Unmarshal(raw, &sin)
|
||||
if sin.Seat == "" {
|
||||
if s.listFault != nil {
|
||||
return nil, s.listFault
|
||||
}
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Seats: s.trunkSeats})
|
||||
}
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
|
||||
case entryVerbKey:
|
||||
var in meshcatalog.EntryInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
s.mu.Lock()
|
||||
s.entries = append(s.entries, in.Fingerprint)
|
||||
s.mu.Unlock()
|
||||
commit := map[string]string{"src2:e1": commitE1, "src2:e2": commitE2}[in.Fingerprint]
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: engine(in.Fingerprint, commit)})
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
|
||||
away := func(machine string) bool { return machine == "shanks" }
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
|
||||
s.r = &compositionRun{runs: s.runs,
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
|
||||
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
|
||||
trunk: func() (mcc.Half, bool) {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
|
||||
},
|
||||
running: func(node, module string) (ne.Running, bool) {
|
||||
fp, ok := running[node]
|
||||
return ne.Running{Node: node, Module: module, Fingerprint: fp}, ok && module == "engine"
|
||||
},
|
||||
put: func(_ context.Context, a buildstep.StepAsk) error {
|
||||
body, err := keeper.body(a.Run.Input.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := composer.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node := req.Machines[0]
|
||||
s.mu.Lock()
|
||||
s.asks[node], s.requests[node] = a, req
|
||||
s.mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
override, set := composer.Machine{}, false
|
||||
if s.verdict != nil {
|
||||
override, set = s.verdict(node)
|
||||
}
|
||||
if set {
|
||||
m = override
|
||||
} else if node == s.unjudged {
|
||||
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
|
||||
} else if node == "g14" {
|
||||
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
|
||||
}
|
||||
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
|
||||
s.mu.Lock()
|
||||
s.puts++
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
|
||||
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
|
||||
// depth compose with the one machine and one case `change`, its manifests and seats from the step's revision, no verdicts (H-A-112); the
|
||||
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
|
||||
// trunk's, or the revision's for a machine whose node-engine is not heard running (A-451); the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
|
||||
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
out := s.r.run(context.Background(), "ask-step", step)
|
||||
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
|
||||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
|
||||
out.Warnings[1] != "g14: the composer names no culprits" {
|
||||
t.Fatalf("the step came to %+v", out)
|
||||
}
|
||||
if len(s.asks) != 2 {
|
||||
t.Fatalf("asked %d runs, want one per machine not away", len(s.asks))
|
||||
}
|
||||
a, req := s.asks["ace"], s.requests["ace"]
|
||||
if a.Kind != buildstep.AskCheck || a.Language != buildstep.AskGo || a.Asker != runAsker || a.Run == nil || len(a.Run.Heads) != 0 ||
|
||||
a.Run.Input.Verb != "seat:mesh-wants.input" || a.Run.Program != (buildstep.RunProgram{Repository: "novox/mesh-host", Commit: commitE1, Dir: ".", Package: composer.Package}) {
|
||||
t.Errorf("the run ask of ace %+v %+v", a, a.Run)
|
||||
}
|
||||
if s.asks["g14"].Run.Program.Commit != commitE2 {
|
||||
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
|
||||
}
|
||||
// Each composer is built against the pins of the node-engine entry it is built from, a pin with no version left out
|
||||
// (H-A-203, H-A-192).
|
||||
for node, fp := range map[string]string{"ace": "src2:e1", "g14": "src2:e2"} {
|
||||
if got := s.asks[node].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25-"+fp {
|
||||
t.Errorf("%s's composer is built against the pins %v; want its entry's", node, got)
|
||||
}
|
||||
}
|
||||
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("the request of ace %+v", req)
|
||||
}
|
||||
c := req.Cases[0]
|
||||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" {
|
||||
t.Errorf("the case %+v", c.Manifests)
|
||||
}
|
||||
if strings.Join(s.entries, " ") == "" {
|
||||
t.Error("no node-engine entry is read through entry")
|
||||
}
|
||||
|
||||
// The revision's node-engine differs from the trunk's: that build composes every machine, and no entry is asked.
|
||||
changed := newRunStand(t, "src2:e3", "src2:e1", "d1")
|
||||
if out := changed.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed {
|
||||
t.Fatalf("the step came to %+v", out)
|
||||
}
|
||||
if changed.asks["ace"].Run.Program.Commit != commitE3 || changed.asks["g14"].Run.Program.Commit != commitE3 || len(changed.entries) != 0 {
|
||||
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
|
||||
}
|
||||
|
||||
// No running of the node-engine is heard on g14: its composer is the revision's entry, and no entry is asked for it (A-451).
|
||||
unheard := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
unheard.r.running = func(node, module string) (ne.Running, bool) {
|
||||
return ne.Running{Node: node, Module: module, Fingerprint: "src2:e1"}, node == "ace" && module == "engine"
|
||||
}
|
||||
if out := unheard.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") {
|
||||
t.Fatalf("the step with g14's node-engine not heard came to %+v", out)
|
||||
}
|
||||
if unheard.asks["g14"].Run == nil || unheard.asks["g14"].Run.Program.Commit != commitE1 {
|
||||
t.Errorf("g14's composer with no running heard is %+v, want the revision's entry", unheard.asks["g14"].Run)
|
||||
}
|
||||
|
||||
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
|
||||
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
|
||||
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "s1") || !strings.Contains(out.Said, "d1") || len(other.asks) != 0 {
|
||||
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
|
||||
}
|
||||
|
||||
// A revision naming no definition (A-234's stand-in): the composer is handed the trunk seat list's definitions, read through
|
||||
// seat-definitions (A-467); a list that cannot be read, or names no seat, ends the step could not run, naming it, with no ask put.
|
||||
bare := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
bare.noDefinitions, bare.trunkSeats = true, []meshcatalog.SeatDefinitionsAnswerSeatsItem{{Seat: "s1", Digest: "d1"}}
|
||||
if out := bare.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
len(bare.requests["ace"].Seats) != 1 || string(bare.requests["ace"].Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("a revision with no definitions came to %+v, seats %v", out, bare.requests["ace"].Seats)
|
||||
}
|
||||
for name, fault := range map[string]error{"unreadable": errors.New("no responder"), "empty": nil} {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.noDefinitions, s.listFault = true, fault
|
||||
if out := s.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "seat list is not read") || len(s.asks) != 0 {
|
||||
t.Errorf("the %s seat list came to %+v, %d asks", name, out, len(s.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// A machine the composer did not judge is could not run, naming why: a step that judged nothing never passes (A-467).
|
||||
nj := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
nj.unjudged = "g14"
|
||||
if out := nj.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "g14 was not judged: the composer was handed no seat list") {
|
||||
t.Errorf("a machine not judged came to %+v", out)
|
||||
}
|
||||
|
||||
// No machine judged: every machine away, or no machine, never passes; each is named with why (A-467a).
|
||||
for name, machines := range map[string][]string{"every machine away": {"shanks"}, "no machine": {}} { // none placed: the inputs say so already
|
||||
want := "no machine was judged"
|
||||
if len(machines) == 0 {
|
||||
want = "has no machine to compose"
|
||||
}
|
||||
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
none.r.inputs.machines = func(string, string) []string { return machines }
|
||||
outcome := meshwants.StepEndedOutcomeCouldNotRun
|
||||
if len(machines) == 0 { // a mover no machine holds: nothing to run, saying why (H-A-179)
|
||||
outcome = meshwants.StepEndedOutcomeNothingToRun
|
||||
}
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != outcome ||
|
||||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
|
||||
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in
|
||||
// the machine's reason (H-A-143, §532).
|
||||
across := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
across.r.newest = func() map[string]mcc.Half {
|
||||
return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}}
|
||||
}
|
||||
across.r.watched = func(module string) []string {
|
||||
if module == "checker" || module == "lidarr" { // no registration names them; lidarr is on main's revision all the same (H-A-144)
|
||||
return nil
|
||||
}
|
||||
return []string{"main", "master"}
|
||||
}
|
||||
across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker", "lidarr"} }
|
||||
if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
!strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") ||
|
||||
!strings.Contains(out.Said, "g14 is assigned lidarr, which no watched branch lists (H-A-143)") {
|
||||
t.Errorf("the step across the watched branches came to %+v", out)
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 4 ||
|
||||
c.Manifests["lidarr"].Raw != nil {
|
||||
t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests)))
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; c.Assignments != nil {
|
||||
t.Errorf("with no assignments held the case composes the input's, it carries %v", c.Assignments)
|
||||
}
|
||||
|
||||
// A stale assignment is left out of the case's assignments as well, said, and the machine composes; a listed module stays in
|
||||
// (H-A-148, ADR 0578 §2). The one set is the watched branches' newest revisions', carried on the run (H-A-152, H-A-153): a module
|
||||
// the candidate drops while the newest still lists it stays in, the change's own fault, and while no newest revision is followed
|
||||
// nothing is left out.
|
||||
dropsWeb := func() map[string]mcc.Half {
|
||||
h := across.r.newest()
|
||||
h["main"] = mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: "main", Revision: 6},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Manifest: map[string]any{"module": "m"}},
|
||||
"web": {Module: "web", Fp: "w1", Manifest: map[string]any{"module": "web"}}}}
|
||||
return h
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string
|
||||
newest func() map[string]mcc.Half // nil: across's
|
||||
want []string // the case's modules of ace; nil: the input's assignments
|
||||
}{
|
||||
{"a stale assignment left out and said, the machine composes", []string{"m", "f1-circuits", "checker"}, nil, []string{"m", "f1-circuits"}},
|
||||
{"listed modules alone: the input's assignments", []string{"m", "f1-circuits"}, nil, nil},
|
||||
{"a module the candidate drops, the newest lists: kept", []string{"m", "f1-circuits", "web"}, dropsWeb, nil},
|
||||
{"no newest revision followed: nothing left out", []string{"m", "checker"}, func() map[string]mcc.Half { return map[string]mcc.Half{} }, nil},
|
||||
} {
|
||||
left := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
left.r.newest, left.r.watched = across.r.newest, across.r.watched
|
||||
if tc.newest != nil {
|
||||
left.r.newest = tc.newest
|
||||
}
|
||||
left.r.assigned = func(string) []string { return tc.assigned }
|
||||
left.r.assignment = func(node string) (mma.Assignments, bool) {
|
||||
a := mma.Assignments{Node: node}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
return a, true
|
||||
}
|
||||
left.verdict = func(string) (composer.Machine, bool) { return composer.Machine{Composes: true}, true }
|
||||
raised := map[string][]byte{}
|
||||
left.r.findings = assign.Findings{Put: putInto(raised)}
|
||||
out := left.r.run(context.Background(), "ask-step", step)
|
||||
// H-A-149: the step that leaves an assignment out raises assignment-unlisted, scope delivery, id <node>.<module>, a warning.
|
||||
var f mcseat.Finding
|
||||
_ = json.Unmarshal(raised[assign.UnlistedKey("ace", "checker")], &f)
|
||||
if want := tc.want != nil; want != (f.Kind == assign.KindAssignmentUnlisted && f.Scope == "delivery" && f.ID == "ace.checker" &&
|
||||
f.Severity == mcseat.FindingSeverityWarning) || (!want && len(raised) != 0) {
|
||||
t.Errorf("%s: raised %v", tc.name, raised)
|
||||
}
|
||||
said := "ace is assigned checker, which no watched branch lists (H-A-143)"
|
||||
if out.Outcome != meshwants.StepEndedOutcomePassed || (tc.want != nil) != slices.Contains(out.Warnings, said) {
|
||||
t.Errorf("%s: the step came to %+v", tc.name, out)
|
||||
}
|
||||
if run, _, _, _ := left.runs.read(runKeyOf("main.7", "ace")); (tc.want != nil) != slices.Equal(run.Unlisted, []string{"checker"}) ||
|
||||
(tc.want == nil && run.Unlisted != nil) {
|
||||
t.Errorf("%s: the run carries the unlisted %v", tc.name, run.Unlisted)
|
||||
}
|
||||
raw, ok := left.requests["ace"].Cases[0].Assignments["ace"]
|
||||
if tc.want == nil {
|
||||
if ok {
|
||||
t.Errorf("%s: the case carries assignments %s", tc.name, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
a, err := mmac.DecodeAssignments(raw)
|
||||
var got []string
|
||||
for _, m := range a.Modules {
|
||||
got = append(got, m.Module)
|
||||
}
|
||||
if err != nil || !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: the case composes ace with %v (%v), want %v", tc.name, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// At the inputs state's cap the ask is refused, naming the store (§354).
|
||||
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
|
||||
if out := full.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
!strings.Contains(out.Said, "the state inputs is at its cap") || len(full.asks) != 0 {
|
||||
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
|
||||
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
|
||||
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
faults int // consecutive faults of the `revision` read before it answers
|
||||
fault error // the fault
|
||||
steps int
|
||||
want string // the outcome of the last step
|
||||
revisionCalls int
|
||||
waits int
|
||||
}{
|
||||
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
|
||||
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
|
||||
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
|
||||
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
inner := s.r.call
|
||||
var mu sync.Mutex
|
||||
calls, faults := 0, tc.faults
|
||||
s.r.call = func(key string, body any) (json.RawMessage, error) {
|
||||
if key == mcc.RevisionVerb {
|
||||
mu.Lock()
|
||||
calls++
|
||||
f := faults > 0
|
||||
faults--
|
||||
mu.Unlock()
|
||||
if f {
|
||||
return nil, tc.fault
|
||||
}
|
||||
}
|
||||
return inner(key, body)
|
||||
}
|
||||
clock := time.Unix(1000, 0)
|
||||
var waits []time.Duration
|
||||
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
|
||||
s.r.sleep = func(_ context.Context, d time.Duration) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
waits = append(waits, d)
|
||||
clock = clock.Add(d)
|
||||
return nil
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, tc.steps)
|
||||
for i := range outs {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
|
||||
}
|
||||
wg.Wait()
|
||||
for _, o := range outs {
|
||||
if o.Outcome != tc.want {
|
||||
t.Fatalf("a step came to %+v, want %s", o, tc.want)
|
||||
}
|
||||
}
|
||||
if tc.waits == -1 { // never answers: bounded, growing waits
|
||||
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
|
||||
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
|
||||
}
|
||||
var total time.Duration
|
||||
for _, w := range waits {
|
||||
total += w
|
||||
}
|
||||
if total > readsBound {
|
||||
t.Errorf("read again for %s, bound %s", total, readsBound)
|
||||
}
|
||||
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
|
||||
t.Errorf("the step said %q", outs[0].Said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if calls != tc.revisionCalls || len(waits) != tc.waits {
|
||||
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
|
||||
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
|
||||
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
|
||||
// own runs, and a run that could not run is never read again.
|
||||
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, 3)
|
||||
for i, m := range []string{"m", "m2", "m3"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
|
||||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
|
||||
t.Errorf("step %d names the runs %+v", i, o.Shared)
|
||||
}
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
|
||||
t.Errorf("step %d came to %+v, want both machines read", i, o)
|
||||
}
|
||||
}
|
||||
if s.puts != 2 {
|
||||
t.Fatalf("%d run asks put, want one per machine", s.puts)
|
||||
}
|
||||
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
|
||||
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
|
||||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
|
||||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
|
||||
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
|
||||
}
|
||||
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
|
||||
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
|
||||
}
|
||||
// Another candidate (a run on request) starts its own runs.
|
||||
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
|
||||
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
|
||||
}
|
||||
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
|
||||
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
|
||||
put, refused := f.r.put, 0
|
||||
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
|
||||
if refused == 0 {
|
||||
refused++
|
||||
return errors.New("no stream")
|
||||
}
|
||||
return put(ctx, a)
|
||||
}
|
||||
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
|
||||
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
|
||||
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
|
||||
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
|
||||
// try is could not run for its own step; a run that could not run is could not run.
|
||||
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
|
||||
at := time.Now()
|
||||
runOf := func(verdict string) meshwants.CompositionRun {
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
|
||||
}
|
||||
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
|
||||
for _, tc := range []struct {
|
||||
name, module string
|
||||
run meshwants.CompositionRun
|
||||
failed, couldNo string
|
||||
warnings []string
|
||||
unlisted []string
|
||||
}{
|
||||
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil},
|
||||
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil},
|
||||
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil},
|
||||
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil},
|
||||
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil},
|
||||
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
|
||||
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil},
|
||||
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
|
||||
[]string{"ace holds a: not tried: the search reached its bound"}, nil},
|
||||
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil},
|
||||
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil},
|
||||
// H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes.
|
||||
{"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`),
|
||||
"ace does not compose and blames no module: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"does not compose, blames a module, an unlisted module", "a", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[{"module":"a","problem":"nothing provides \"photos\""}]}`),
|
||||
"ace holds a: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"another module blamed, an unlisted module", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"x"}]}`), "", "",
|
||||
[]string{"ace holds a: x", "ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
{"could not run, an unlisted module", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"},
|
||||
"", "ace: gone; ace is assigned photos, which no watched branch lists (H-A-143)", nil, []string{"photos"}},
|
||||
{"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "",
|
||||
[]string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
run := tc.run
|
||||
run.Unlisted = tc.unlisted // the set the run's start wrote (H-A-153)
|
||||
e := judgeRun(tc.module, run)
|
||||
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
|
||||
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
|
||||
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
|
||||
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
|
||||
// step passed keeps its verdict.
|
||||
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var mu sync.Mutex
|
||||
fixed := false
|
||||
s.verdict = func(node string) (composer.Machine, bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
m.Blamed = []struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{}
|
||||
if node == "g14" && !fixed {
|
||||
m.Composes, m.Reason = false, "two modules own one path"
|
||||
m.Blamed = append(m.Blamed, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"a", "/etc/x"}, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"d", "/etc/x"})
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
ends := make(chan meshwants.StepEnded, 16)
|
||||
msgIDs := map[string]string{} // the message id of each step's latest end
|
||||
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
|
||||
say: func(event string, body any, msgID string) error {
|
||||
if e, ok := body.(meshwants.StepEnded); ok {
|
||||
mu.Lock()
|
||||
msgIDs[e.ID] = msgID
|
||||
mu.Unlock()
|
||||
ends <- e
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
move := steps.move(context.Background())
|
||||
s.runs.moved.Store(&move)
|
||||
step := func(m string, rerun bool) []byte {
|
||||
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
|
||||
return raw
|
||||
}
|
||||
got := map[string]meshwants.StepEnded{}
|
||||
for _, m := range []string{"a", "b", "d"} {
|
||||
pullOne(steps, "ask-"+m, step(m, false))
|
||||
e := <-ends
|
||||
got[e.Module] = e
|
||||
}
|
||||
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the first ends: %+v", got)
|
||||
}
|
||||
former := got["d"].Shared[1].Ask
|
||||
mu.Lock()
|
||||
fixed = true
|
||||
mu.Unlock()
|
||||
pullOne(steps, "ask-a2", step("a", true))
|
||||
moved := map[string]meshwants.StepEnded{}
|
||||
for range 2 {
|
||||
select {
|
||||
case e := <-ends:
|
||||
moved[e.ID] = e
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends after the rerun: %+v", moved)
|
||||
}
|
||||
}
|
||||
a, d := moved["ask-a2"], moved["ask-d"]
|
||||
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
|
||||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
|
||||
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
|
||||
}
|
||||
mu.Lock()
|
||||
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
|
||||
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
|
||||
t.Errorf("the message ids of the ends: %v", msgIDs)
|
||||
}
|
||||
mu.Unlock()
|
||||
if s.puts != 3 {
|
||||
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
|
||||
}
|
||||
select {
|
||||
case e := <-ends:
|
||||
t.Errorf("a reader that passed moved: %+v", e)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
marks := map[string]bool{}
|
||||
for _, c := range run.Readers {
|
||||
marks[c.ID] = c.Rerun
|
||||
}
|
||||
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
|
||||
t.Errorf("the new run's readers %+v", run.Readers)
|
||||
}
|
||||
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
wake := s.runs.wake()
|
||||
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-wake:
|
||||
case <-deadline:
|
||||
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its
|
||||
// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the
|
||||
// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision.
|
||||
func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) {
|
||||
entry := func(module, fp string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}}
|
||||
}
|
||||
half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half {
|
||||
h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: branch}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, m := range movers {
|
||||
h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m})
|
||||
}
|
||||
for _, e := range entries {
|
||||
h.Entries[e.Module] = e
|
||||
}
|
||||
return h
|
||||
}
|
||||
newest := map[string]mcc.Half{
|
||||
"main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")),
|
||||
"master": half("master", 9, nil, entry("f1-circuits", "f1")),
|
||||
"nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")),
|
||||
}
|
||||
registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}}
|
||||
registrations["engine"], registrations["a"] = []string{"main"}, []string{"main"}
|
||||
watched := func(module string) []string { return registrations[module] } // a module no registration names has none (H-A-144)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
step meshwants.StepRun
|
||||
candidate mcc.Half
|
||||
newest map[string]mcc.Half
|
||||
want map[string]string // module -> fingerprint
|
||||
}{
|
||||
{"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}},
|
||||
{"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2},
|
||||
half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}},
|
||||
{"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1},
|
||||
half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]},
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}},
|
||||
{"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}},
|
||||
{"a module no registration names is not composed", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2"), entry("checker", "c1")), nil, map[string]string{"a": "a2"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := acrossBranches(tc.step, tc.candidate, tc.newest, watched)
|
||||
fps := map[string]string{}
|
||||
for name, e := range got.Entries {
|
||||
fps[name] = e.Fp
|
||||
}
|
||||
if !maps.Equal(fps, tc.want) {
|
||||
t.Errorf("composed from %v, want %v", fps, tc.want)
|
||||
}
|
||||
if got.Revision.Revision != tc.candidate.Revision.Revision || len(got.Revision.Definitions) != 1 || got.Revision.Definitions[0].Digest != tc.step.Branch {
|
||||
t.Errorf("the half keeps revision %d with definitions %v, want the candidate's %d with its own", got.Revision.Revision, got.Revision.Definitions,
|
||||
tc.candidate.Revision.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-149: the pass over the assignments clears assignment-unlisted when the assignment is gone or one of the module's
|
||||
// watched branches lists it again, and keeps it while it stands; another finding is left alone.
|
||||
func TestThePassClearsAnUnlistedAssignmentsFinding(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string // novox's modules
|
||||
listed bool
|
||||
kept bool
|
||||
}{
|
||||
{"still assigned and listed nowhere: kept", []string{"checker"}, false, true},
|
||||
{"the assignment gone: cleared", []string{"web"}, false, false},
|
||||
{"listed again: cleared", []string{"checker"}, true, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := newWorld()
|
||||
a := mma.Assignments{V: keys.Version, Node: "novox"}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
putAssignments(w, a)
|
||||
findings := map[string][]byte{}
|
||||
f, _ := assign.UnlistedFinding("novox", "checker", time.Now())
|
||||
_ = putInto(findings)(f)
|
||||
other, _ := assign.UnplacedFinding("g14", time.Now())
|
||||
_ = putInto(findings)(other)
|
||||
p := &passer{w: w, logf: t.Logf, listed: func(string) bool { return tc.listed }, findings: assign.Findings{Put: putInto(findings),
|
||||
Delete: func(k string) error { delete(findings, k); return nil },
|
||||
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(findings)), nil }}}
|
||||
p.clearUnlisted()
|
||||
if _, ok := findings[assign.UnlistedKey("novox", "checker")]; ok != tc.kept {
|
||||
t.Errorf("the finding kept %v, want %v", ok, tc.kept)
|
||||
}
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok {
|
||||
t.Error("another finding is cleared")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,916 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
|
||||
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
|
||||
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
|
||||
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
|
||||
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
|
||||
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
|
||||
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
|
||||
|
||||
const (
|
||||
// runsSwept is how many of the oldest ended keys each new run judges.
|
||||
runsSwept = 20
|
||||
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
|
||||
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
|
||||
runBoundPrefix = "bound."
|
||||
runUntakenPrefix = "untaken."
|
||||
)
|
||||
|
||||
// runKeyOf is the key of a candidate's composition run of a machine (§719).
|
||||
func runKeyOf(candidate, machine string) string {
|
||||
return keysPart(candidate) + "." + keys.EscapeName(machine)
|
||||
}
|
||||
|
||||
// keysPart is a name as a key's part.
|
||||
func keysPart(name string) string { return keys.EscapeName(name) }
|
||||
|
||||
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
|
||||
func isRunKey(key string) bool {
|
||||
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
|
||||
}
|
||||
|
||||
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
|
||||
type runSettings struct {
|
||||
// capBytes is composition-runs-cap-mib in bytes.
|
||||
capBytes int64
|
||||
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
|
||||
kept, stepBound, untaken time.Duration
|
||||
// blameTries is blame-tries, carried in each run's request (§721).
|
||||
blameTries int
|
||||
}
|
||||
|
||||
// defaultRunSettings are the settings' defaults, as the manifest gives them.
|
||||
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
|
||||
|
||||
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
|
||||
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
|
||||
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
|
||||
func runSettingsOf(getenv func(string) string) (runSettings, error) {
|
||||
s := defaultRunSettings
|
||||
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if mib > 0 {
|
||||
s.capBytes = int64(mib) << 20
|
||||
}
|
||||
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if tries > 0 {
|
||||
s.blameTries = tries
|
||||
}
|
||||
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
|
||||
d, err := durationFile(getenv(env))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if d > 0 {
|
||||
*to = d
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
|
||||
func durationFile(path string) (time.Duration, error) {
|
||||
if path == "" {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return 0, nil
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// errRunChanged is a compare-and-set of a run's key that met another write.
|
||||
var errRunChanged = errors.New("the composition run's key changed")
|
||||
|
||||
// runsKV is the state composition-runs: the bus bucket, or a test's.
|
||||
type runsKV interface {
|
||||
// get is a key's value and revision; nil for a key not there.
|
||||
get(key string) ([]byte, uint64, error)
|
||||
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
|
||||
create(key string, value []byte) (uint64, error)
|
||||
update(key string, value []byte, revision uint64) (uint64, error)
|
||||
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
|
||||
// of the state is made from a value read before its latest change.
|
||||
delAt(key string, revision uint64) error
|
||||
keys() ([]string, error)
|
||||
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
|
||||
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
|
||||
// answers the revision written.
|
||||
arm(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// used is how many bytes the state holds.
|
||||
used() (int64, error)
|
||||
}
|
||||
|
||||
// busRuns is runsKV over the bus bucket.
|
||||
type busRuns struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b busRuns) get(key string) ([]byte, uint64, error) {
|
||||
e, err := b.kv.Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (b busRuns) create(key string, value []byte) (uint64, error) {
|
||||
rev, err := b.kv.Create(b.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := b.kv.Update(b.ctx, key, value, revision)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) delAt(key string, revision uint64) error {
|
||||
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return errRunChanged
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busRuns) keys() ([]string, error) {
|
||||
l, err := b.kv.ListKeys(b.ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b.js == nil {
|
||||
return 0, errors.New("no bus to arm a bound on")
|
||||
}
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject := "$KV." + b.kv.Bucket() + "." + key
|
||||
for {
|
||||
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var last uint64
|
||||
m, err := st.GetLastMsgForSubject(b.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b busRuns) used() (int64, error) {
|
||||
st, err := b.kv.Status(b.ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(st.Bytes()), nil
|
||||
}
|
||||
|
||||
// compositionRuns keeps the composition runs in the state composition-runs.
|
||||
type compositionRuns struct {
|
||||
kv runsKV
|
||||
set runSettings
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
|
||||
changed chan struct{}
|
||||
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
|
||||
// being written at that moment, so every lookup takes these beside it (keys).
|
||||
seen map[string]bool
|
||||
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
|
||||
// nil says or drops nothing.
|
||||
say publisher
|
||||
drop func(ask, why string) error
|
||||
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
|
||||
// Set once the steps are served, after the hub starts.
|
||||
moved atomic.Pointer[func(run meshwants.CompositionRun)]
|
||||
}
|
||||
|
||||
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
|
||||
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
|
||||
func (r *compositionRuns) heard() {
|
||||
r.mu.Lock()
|
||||
close(r.changed)
|
||||
r.changed = make(chan struct{})
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// saw keeps a key the state's watch handed: standing, or deleted or expired.
|
||||
func (r *compositionRuns) saw(key string, standing bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.seen == nil {
|
||||
r.seen = map[string]bool{}
|
||||
}
|
||||
if standing {
|
||||
r.seen[key] = true
|
||||
} else {
|
||||
delete(r.seen, key)
|
||||
}
|
||||
}
|
||||
|
||||
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
|
||||
// is skipped by its reader.
|
||||
func (r *compositionRuns) keys() ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
for key := range r.seen {
|
||||
all = append(all, key)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
slices.Sort(all)
|
||||
return slices.Compact(all), nil
|
||||
}
|
||||
|
||||
// wake is closed at the next change heard.
|
||||
func (r *compositionRuns) wake() <-chan struct{} {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.changed
|
||||
}
|
||||
|
||||
// read is a run's value and revision, or false.
|
||||
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(key)
|
||||
if err != nil || raw == nil {
|
||||
return meshwants.CompositionRun{}, 0, false, err
|
||||
}
|
||||
var run meshwants.CompositionRun
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
|
||||
}
|
||||
return run, rev, true, nil
|
||||
}
|
||||
|
||||
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
|
||||
// It answers the value written or left, and false for a key not there.
|
||||
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok {
|
||||
return run, ok, err
|
||||
}
|
||||
if !f(&run) {
|
||||
return run, true, nil
|
||||
}
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
r.heard()
|
||||
return run, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fill is the state's fill against its cap.
|
||||
func (r *compositionRuns) fill() (meshwants.Fill, error) {
|
||||
at := r.now().UTC()
|
||||
used, err := r.kv.used()
|
||||
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
|
||||
}
|
||||
|
||||
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
|
||||
// value as it stood.
|
||||
type joined struct {
|
||||
key string
|
||||
started bool
|
||||
run meshwants.CompositionRun
|
||||
}
|
||||
|
||||
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
|
||||
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
|
||||
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
|
||||
// cap no run is started, an error naming the fill.
|
||||
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
return r.joinAt(runKeyOf(candidate, machine), reader, candidate, machine, ask)
|
||||
}
|
||||
|
||||
// joinAt is join of the run under key: a machine's composition run, or a revision's validate run with no machine (H-A-108).
|
||||
func (r *compositionRuns) joinAt(key string, reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
|
||||
if hasReader(run.Readers, reader.ID) {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
run.Readers = append(run.Readers, reader)
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) {
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
|
||||
Readers: []meshwants.CompositionReader{reader}}
|
||||
raw, err := json.Marshal(fresh)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok {
|
||||
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
|
||||
} else {
|
||||
_, err = r.kv.create(key, raw)
|
||||
}
|
||||
if errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
|
||||
func (r *compositionRuns) room() error {
|
||||
if err := r.sweep(); err != nil {
|
||||
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
|
||||
}
|
||||
f, err := r.fill()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
if f.Bytes >= f.Cap {
|
||||
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
|
||||
f.Cap>>20, f.Bytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
|
||||
func (r *compositionRuns) sweep() error {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type endedAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var ended []endedAt
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended == nil {
|
||||
continue
|
||||
}
|
||||
ended = append(ended, endedAt{key, *run.Ended})
|
||||
}
|
||||
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
|
||||
if len(ended) > runsSwept {
|
||||
ended = ended[:runsSwept]
|
||||
}
|
||||
var errs []error
|
||||
for _, e := range ended {
|
||||
if r.now().Sub(e.at) >= r.set.kept {
|
||||
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
|
||||
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
|
||||
}))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
|
||||
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
|
||||
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok || !gone(run) {
|
||||
return err
|
||||
}
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
|
||||
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
|
||||
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
|
||||
var errs []error
|
||||
for _, prefix := range prefixes {
|
||||
for {
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if ok && run.Ask != ask {
|
||||
break // a newer run holds the key: its bounds are its own
|
||||
}
|
||||
held, rev, standing, err := r.boundOf(prefix + key)
|
||||
if err != nil || !standing {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if held != ask {
|
||||
break // the bound of another ask (A-470n): not this run's to delete
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
|
||||
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
|
||||
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
|
||||
keys, err := r.keysOfStep(step, id)
|
||||
if err != nil {
|
||||
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
|
||||
return
|
||||
}
|
||||
at = at.UTC()
|
||||
for _, key := range keys {
|
||||
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
c.Readers[i].Ended = &at
|
||||
return true
|
||||
}); err != nil {
|
||||
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keysOfStep are the keys of the runs the step asked under id reads: its revision's validate run for a validate step, its
|
||||
// candidate's runs naming it for a composition step.
|
||||
func (r *compositionRuns) keysOfStep(step meshwants.StepRun, id string) ([]string, error) {
|
||||
if step.Step == meshwants.StepRunStepValidate {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{key}, nil
|
||||
}
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.keysOf(candidate, id)
|
||||
}
|
||||
|
||||
// hasReader is whether readers holds the step asked under id.
|
||||
func hasReader(readers []meshwants.CompositionReader, id string) bool {
|
||||
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
}
|
||||
|
||||
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
|
||||
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
|
||||
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
|
||||
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
|
||||
}
|
||||
|
||||
// stepRunOf is a reader's StepRun again, for its step-ended.
|
||||
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
|
||||
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
|
||||
}
|
||||
|
||||
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
|
||||
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
|
||||
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
if ready != nil {
|
||||
ready()
|
||||
ready = nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
|
||||
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
|
||||
expired(e.Key())
|
||||
}
|
||||
r.heard()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
|
||||
// false when the key holds another run or the run has ended already.
|
||||
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
ended := false
|
||||
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
at := r.now().UTC()
|
||||
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
|
||||
ended = true
|
||||
return true
|
||||
})
|
||||
return ended, err
|
||||
}
|
||||
|
||||
// keyOfAsk is the key of the run whose ask is ask, or none.
|
||||
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err == nil && ok && run.Ask == ask {
|
||||
return key, run, nil
|
||||
}
|
||||
}
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
|
||||
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
|
||||
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
|
||||
// names is another asker's, or of a run gone, and changes nothing.
|
||||
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
|
||||
if e.Asker != runAsker || e.Ask == "" {
|
||||
return nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(e.Ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
|
||||
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
|
||||
return nil
|
||||
}
|
||||
node := run.Machine
|
||||
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
|
||||
var verdict map[string]any
|
||||
if e.Outcome != buildstep.OutcomePassed {
|
||||
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
|
||||
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
|
||||
said = node + ": " + err.Error()
|
||||
} else if isValidateRun(run) {
|
||||
if verdict, said = validateRunVerdict(answer); said == "" {
|
||||
outcome = meshwants.CompositionRunOutcomePassed
|
||||
}
|
||||
} else if verdict, said = runVerdict(node, answer); said == "" {
|
||||
outcome = meshwants.CompositionRunOutcomePassed
|
||||
}
|
||||
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
|
||||
return err
|
||||
}
|
||||
|
||||
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
|
||||
// stands then; a key gone could not run.
|
||||
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
|
||||
out := map[string]meshwants.CompositionRun{}
|
||||
for {
|
||||
wake := r.wake()
|
||||
open := false
|
||||
for _, key := range keys {
|
||||
run, _, ok, err := r.read(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
open = true
|
||||
continue
|
||||
case !ok:
|
||||
at := r.now().UTC()
|
||||
candidate, machine, _ := strings.Cut(key, ".")
|
||||
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
|
||||
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
|
||||
case run.Ended == nil:
|
||||
open = true
|
||||
}
|
||||
out[key] = run
|
||||
}
|
||||
if !open {
|
||||
return out
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return out
|
||||
case <-wake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unescape is a key's part as a name again; a part that does not read stays as it is.
|
||||
func unescape(part string) string {
|
||||
if name, err := keys.UnescapeName(part); err == nil {
|
||||
return name
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
|
||||
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
|
||||
out := []meshwants.Shared{}
|
||||
for _, key := range keys {
|
||||
run, ok := runs[key]
|
||||
if !ok {
|
||||
if run, _, ok, _ = r.read(key); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
|
||||
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
|
||||
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
|
||||
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
var errs []error
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
var left []string
|
||||
var leaving []meshwants.CompositionReader
|
||||
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
left, leaving = nil, nil
|
||||
keep := c.Readers[:0:0]
|
||||
for _, reader := range c.Readers {
|
||||
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
|
||||
left = append(left, reader.ID)
|
||||
leaving = append(leaving, reader)
|
||||
continue
|
||||
}
|
||||
keep = append(keep, reader)
|
||||
}
|
||||
c.Readers = keep
|
||||
return len(left) > 0
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if !ok || len(left) == 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, left...)
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
|
||||
r.sayWaits(run, leaving, "superseded")
|
||||
}
|
||||
if stop != nil {
|
||||
stop(left) // the steps that left say no end, the run's included
|
||||
}
|
||||
if run.Ended != nil || len(run.Readers) > 0 {
|
||||
continue
|
||||
}
|
||||
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, why); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return ids, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
|
||||
func passedOn(module string, run meshwants.CompositionRun) bool {
|
||||
e := judgeVerdict(module, run) // the unlisted modules change only the words, never whether it passed
|
||||
return e.failed == "" && e.couldNot == ""
|
||||
}
|
||||
|
||||
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
|
||||
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
|
||||
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
|
||||
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
|
||||
})
|
||||
}
|
||||
|
||||
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
|
||||
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
|
||||
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
|
||||
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
|
||||
// not (A-470d).
|
||||
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
key := runKeyOf(candidate, machine)
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if !ok {
|
||||
return r.join(reader, candidate, machine, ask)
|
||||
}
|
||||
at := formerOf(run.Readers, reader.Pipeline)
|
||||
ended := run.Ended != nil
|
||||
keep := ended && at >= 0 && passedOn(reader.Module, run)
|
||||
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
|
||||
if keep || read {
|
||||
if at >= 0 && run.Readers[at].ID == reader.ID {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
var former []meshwants.CompositionReader
|
||||
if at >= 0 {
|
||||
former = []meshwants.CompositionReader{run.Readers[at]}
|
||||
run.Readers[at] = reader
|
||||
} else {
|
||||
run.Readers = append(run.Readers, reader)
|
||||
}
|
||||
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
|
||||
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
|
||||
for i, c := range run.Readers {
|
||||
if i == at {
|
||||
continue
|
||||
}
|
||||
c.Rerun, c.By = !passedOn(c.Module, run), ""
|
||||
if c.Rerun {
|
||||
c.By = reader.By
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, c)
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, reader)
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
|
||||
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// write writes a run's value at its revision.
|
||||
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
-404
@@ -1,404 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
|
||||
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
|
||||
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
|
||||
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
|
||||
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
|
||||
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
|
||||
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
|
||||
// pass and moves b; a superseded step leaves
|
||||
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
|
||||
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
|
||||
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
|
||||
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const commit = "1111111111111111111111111111111111111111"
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
awaitKept(t, b, nodeengine.StreamRunning, 2)
|
||||
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
|
||||
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
|
||||
}
|
||||
b.cat.rev[trunkBranch] = 3
|
||||
b.cat.mu.Unlock()
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.place(t, node, m)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
inputsAsked := map[string]int{} // by candidate and node
|
||||
var drops []string
|
||||
respond := func(subject string, f func(m *nats.Msg) any) {
|
||||
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
|
||||
node := strings.Split(m.Subject, ".")[5]
|
||||
var in nodeengine.ComposeCandidateInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
inputsAsked[in.Candidate+" "+node]++
|
||||
mu.Unlock()
|
||||
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
|
||||
})
|
||||
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
|
||||
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
|
||||
var in nodebuildgo.DropInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
drops = append(drops, in.ID)
|
||||
mu.Unlock()
|
||||
return map[string]any{}
|
||||
})
|
||||
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
|
||||
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
|
||||
// repeated message id is never heard (A-470q).
|
||||
ended := make(chan meshwants.StepEnded, 32)
|
||||
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
|
||||
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard, err := runner.Consume(func(m jetstream.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data(), &e)
|
||||
_ = m.Ack()
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(heard.Stop)
|
||||
subscribed(t, nc)
|
||||
|
||||
from := b.journal.mark()
|
||||
_, stop := b.startStoppable(t)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
|
||||
select {
|
||||
case <-b.seatListRead:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
|
||||
AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
step := func(id, module, run string, rerun bool) {
|
||||
t.Helper()
|
||||
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
|
||||
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
|
||||
if run != "" {
|
||||
s.Pipeline = "run-" + run + "." + module
|
||||
}
|
||||
if rerun {
|
||||
s.By = "op@n1"
|
||||
}
|
||||
raw, _ := json.Marshal(s)
|
||||
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Data = raw
|
||||
if err := nc.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
type asked struct {
|
||||
ask buildstep.StepAsk
|
||||
machine string
|
||||
}
|
||||
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
|
||||
asks := func(n int) []asked {
|
||||
t.Helper()
|
||||
var out []asked
|
||||
for range n {
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil {
|
||||
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
|
||||
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
||||
}
|
||||
out = append(out, asked{a, req.Machines[0]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
noMoreAsks := func(what string) {
|
||||
t.Helper()
|
||||
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
|
||||
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
|
||||
}
|
||||
}
|
||||
say := func(event string, node string, body any) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(body)
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer := func(a asked, machine string) {
|
||||
t.Helper()
|
||||
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
|
||||
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
|
||||
}
|
||||
ends := func(n int, what string) map[string]meshwants.StepEnded {
|
||||
t.Helper()
|
||||
out := map[string]meshwants.StepEnded{}
|
||||
for len(out) < n {
|
||||
select {
|
||||
case e := <-ended:
|
||||
out[e.ID] = e
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
runKeys := func(candidate string) []string {
|
||||
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
|
||||
var out []string
|
||||
for _, k := range all {
|
||||
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// readers waits, through a watch of the key, until the run of key has n readers.
|
||||
readers := func(key string, n int) {
|
||||
t.Helper()
|
||||
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
var run meshwants.CompositionRun
|
||||
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the run %s has not %d readers", key, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
|
||||
composes := `{"composes":true,"blamed":[]}`
|
||||
|
||||
// One candidate, three steps, two machines: two runs.
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
step("step-"+m, m, "", false)
|
||||
}
|
||||
first := asks(2)
|
||||
noMoreAsks("three steps of one candidate")
|
||||
for _, a := range first {
|
||||
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
|
||||
}
|
||||
got := ends(3, "the three steps")
|
||||
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
|
||||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
|
||||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
|
||||
t.Fatalf("the steps of one candidate: %+v", got)
|
||||
}
|
||||
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
|
||||
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
|
||||
}
|
||||
|
||||
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
|
||||
step("step-r1", "a", "r1", false)
|
||||
for _, a := range asks(2) {
|
||||
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
|
||||
}
|
||||
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
|
||||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
|
||||
t.Fatalf("a step of another candidate: %+v", e)
|
||||
}
|
||||
|
||||
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
|
||||
from = b.journal.mark()
|
||||
step("step-r2", "a", "r2", false)
|
||||
bound := asks(2)
|
||||
var n1 asked
|
||||
for _, a := range bound {
|
||||
if a.machine == "n1" {
|
||||
n1 = a
|
||||
}
|
||||
}
|
||||
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
|
||||
for i := range 3 { // past the 2 s bound, each beat within it
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a run saying its progress ended: %+v", e)
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
|
||||
runs := b.buckets[buckets.CompositionRuns()]
|
||||
all, _ := runs.Keys(b.ctx)
|
||||
held := ""
|
||||
for _, k := range all {
|
||||
if strings.HasPrefix(k, runBoundPrefix) {
|
||||
if e, err := runs.Get(b.ctx, k); err == nil {
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
held += v.Ask
|
||||
}
|
||||
}
|
||||
}
|
||||
if held != n1.ask.ID {
|
||||
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
|
||||
}
|
||||
}
|
||||
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
|
||||
}
|
||||
e := ends(1, "the bounds")["step-r2"]
|
||||
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
|
||||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
|
||||
t.Fatalf("past the bounds: %+v", e)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := len(drops)
|
||||
mu.Unlock()
|
||||
if dropped != 2 {
|
||||
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
|
||||
}
|
||||
answer(n1, composes) // heard after the bound: ignored and journaled
|
||||
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
|
||||
|
||||
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
|
||||
step("step-a2", "a", "", true)
|
||||
again := asks(1)
|
||||
noMoreAsks("a rerun of a")
|
||||
if again[0].machine != "n2" {
|
||||
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
|
||||
}
|
||||
answer(again[0], composes)
|
||||
moved := ends(2, "the rerun")
|
||||
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
|
||||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
|
||||
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
|
||||
}
|
||||
|
||||
// A superseded step leaves the runs; its other reader goes on.
|
||||
step("step-r3a", "a", "r3", false)
|
||||
step("step-r3b", "b", "r3", false)
|
||||
sup := asks(2)
|
||||
readers(runKeyOf("run-r3", "n1"), 2)
|
||||
readers(runKeyOf("run-r3", "n2"), 2)
|
||||
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Branch: trunkBranch, Source: 1, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Movers: []mpr.MoverEnded{{Module: "a", Steps: []mpr.StepResult{}}}})
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range sup {
|
||||
answer(a, composes)
|
||||
}
|
||||
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the other reader: %+v", got)
|
||||
}
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a superseded step said its end: %+v", e)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Restarted during a run: the new assigner takes it up by its ask.
|
||||
step("step-r4", "c", "r4", false)
|
||||
restart := asks(2)
|
||||
stop()
|
||||
for _, a := range restart {
|
||||
answer(a, composes)
|
||||
}
|
||||
from = b.journal.mark()
|
||||
b.startStoppable(t)
|
||||
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
|
||||
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
|
||||
t.Fatalf("the step taken up: %+v", e)
|
||||
}
|
||||
|
||||
// compositions-drop: the readers ended naming the drop, the key deleted.
|
||||
step("step-r5", "a", "r5", false)
|
||||
asks(2)
|
||||
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
|
||||
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
|
||||
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
|
||||
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
|
||||
}
|
||||
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
|
||||
t.Fatalf("a reader of the dropped run: %+v", e)
|
||||
}
|
||||
if keys := runKeys("run-r5"); len(keys) != 1 {
|
||||
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
|
||||
}
|
||||
|
||||
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
|
||||
for i := range 2 {
|
||||
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
step("step-r6", "a", "r6", false)
|
||||
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
|
||||
t.Fatalf("at the cap: %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -1,642 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
|
||||
type runsMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
ttl map[string]time.Duration
|
||||
bytes int64
|
||||
n uint64
|
||||
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
|
||||
beforeDel func()
|
||||
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
|
||||
beforeArm func()
|
||||
}
|
||||
|
||||
func (k *runsMem) another() {
|
||||
k.mu.Lock()
|
||||
f := k.beforeDel
|
||||
k.beforeDel = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
func newRunsMem() *runsMem {
|
||||
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
|
||||
}
|
||||
|
||||
func (k *runsMem) get(key string) ([]byte, uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.vals[key], k.revs[key], nil
|
||||
}
|
||||
|
||||
func (k *runsMem) create(key string, v []byte) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; ok {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.revs[key] != rev {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) delAt(key string, rev uint64) error {
|
||||
k.another()
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; !ok {
|
||||
return nil
|
||||
}
|
||||
if k.revs[key] != rev {
|
||||
return errRunChanged
|
||||
}
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *runsMem) keys() ([]string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
var out []string
|
||||
for key := range k.vals {
|
||||
out = append(out, key)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
f := k.beforeArm
|
||||
k.beforeArm = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.n++
|
||||
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
|
||||
func (k *runsMem) expire(key string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
}
|
||||
|
||||
func (k *runsMem) used() (int64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.bytes > 0 {
|
||||
return k.bytes, nil
|
||||
}
|
||||
var n int64
|
||||
for _, v := range k.vals {
|
||||
n += int64(len(v))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
|
||||
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
|
||||
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
|
||||
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
step := func(id, module string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
|
||||
}
|
||||
t.Run("key", func(t *testing.T) {
|
||||
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
|
||||
t.Errorf("the key is %q, want each part escaped as a key's name", k)
|
||||
}
|
||||
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
|
||||
t.Error("a bound key is taken for a run's or a run's for a bound key")
|
||||
}
|
||||
})
|
||||
t.Run("one run for two steps at once", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var wg sync.WaitGroup
|
||||
got := make([]joined, 2)
|
||||
for i, m := range []string{"a", "b"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
got[i] = j
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
runKeys := func() []string {
|
||||
all, _ := kv.keys()
|
||||
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
|
||||
}
|
||||
if got[0].started == got[1].started {
|
||||
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
|
||||
}
|
||||
keys := runKeys()
|
||||
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
|
||||
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
|
||||
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
|
||||
}
|
||||
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
|
||||
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("could not run is never read", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
|
||||
return true
|
||||
})
|
||||
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
|
||||
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
|
||||
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
|
||||
}
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
|
||||
return true
|
||||
})
|
||||
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
|
||||
t.Errorf("a passed run is not read: %+v", j)
|
||||
}
|
||||
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
|
||||
t.Errorf("a step of another candidate reads the first's run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
kv.bytes = 16 << 20
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
|
||||
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
|
||||
t.Errorf("at the cap: %v, want the fill named", err)
|
||||
}
|
||||
})
|
||||
t.Run("kept", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
at := now
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
|
||||
for _, c := range []string{"main.1", "main.2"} {
|
||||
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := now.Add(-25 * time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
|
||||
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
|
||||
recent := now.Add(-time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
|
||||
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, _ := kv.keys()
|
||||
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
|
||||
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
|
||||
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
|
||||
}
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
|
||||
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
|
||||
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
|
||||
}
|
||||
})
|
||||
t.Run("a listing that misses a key being written", func(t *testing.T) {
|
||||
kv := &missingRuns{runsMem: newRunsMem()}
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.hide = runKeyOf("main.3", "n1")
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
|
||||
t.Fatalf("the stand's listing does not hide the key")
|
||||
}
|
||||
r.saw(kv.hide, true) // as the state's watch hands it
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
|
||||
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
|
||||
}
|
||||
r.saw(kv.hide, false)
|
||||
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
|
||||
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
|
||||
}
|
||||
})
|
||||
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
|
||||
// replacing the key, and its untaken bound, stand.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ended time.Time
|
||||
act func(r *compositionRuns, key string) error
|
||||
}{
|
||||
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
|
||||
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
|
||||
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
if !c.ended.IsZero() {
|
||||
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
|
||||
}
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
|
||||
for {
|
||||
raw, rev, _ := kv.get(key)
|
||||
var run meshwants.CompositionRun
|
||||
_ = json.Unmarshal(raw, &run)
|
||||
if run.Ended == nil {
|
||||
at := now
|
||||
run.Ended, run.Outcome = &at, "failed"
|
||||
}
|
||||
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
|
||||
if _, err := kv.update(key, fresh, rev); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
if err := c.act(r, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
|
||||
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
|
||||
}
|
||||
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
|
||||
t.Error("the untaken bound the rerun's run armed is deleted")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("settings", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
|
||||
env := map[string]string{}
|
||||
for name, content := range files {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = p
|
||||
}
|
||||
s, err := runSettingsOf(func(k string) string { return env[k] })
|
||||
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
|
||||
t.Errorf("settings %+v %v", s, err)
|
||||
}
|
||||
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
|
||||
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
|
||||
t.Errorf("a bound of no unit: %v", err)
|
||||
}
|
||||
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
|
||||
t.Errorf("no files: %+v %v, want the defaults", s, err)
|
||||
}
|
||||
})
|
||||
raw, _ := json.Marshal(step("x", "a"))
|
||||
if !strings.Contains(string(raw), `"id":"x"`) {
|
||||
t.Errorf("a reader names no ask id: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
|
||||
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
|
||||
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
|
||||
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
|
||||
// are taken up at the start.
|
||||
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
reader := func(id string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
|
||||
}
|
||||
type said struct{ event, how, module string }
|
||||
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
|
||||
kv := newRunsMem()
|
||||
var saids []said
|
||||
var drops, journal []string
|
||||
var mu sync.Mutex
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
journal = append(journal, fmt.Sprintf(f, a...))
|
||||
mu.Unlock()
|
||||
})
|
||||
r.say = func(event string, body any, _ string) error {
|
||||
w := body.(meshwants.Wait)
|
||||
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
|
||||
t.Errorf("the wait said %+v", w)
|
||||
}
|
||||
saids = append(saids, said{event, w.How, w.Module})
|
||||
return nil
|
||||
}
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
return kv, r, &saids, &drops, &journal
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
t.Run("taken, progress, bound", func(t *testing.T) {
|
||||
kv, r, saids, drops, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
|
||||
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
|
||||
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, untaken, _ := kv.get(runUntakenPrefix + key)
|
||||
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
|
||||
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
before := kv.revs[runBoundPrefix+key]
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
|
||||
t.Fatalf("a step-progress does not arm the bound again: %v", err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
|
||||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
|
||||
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
|
||||
}
|
||||
if got := judgeRun("a", run); got.couldNot != run.Said {
|
||||
t.Errorf("a reader of the run past its bound: %+v", got)
|
||||
}
|
||||
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
|
||||
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
|
||||
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
|
||||
}) {
|
||||
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
|
||||
}
|
||||
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
|
||||
t.Errorf("the bound keys stay past the run's end: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("untaken", func(t *testing.T) {
|
||||
kv, r, saids, drops, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
|
||||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
|
||||
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
|
||||
}
|
||||
r.expired(runUntakenPrefix + key) // a run ended takes nothing
|
||||
if len(*drops) != 1 {
|
||||
t.Errorf("an ended run is dropped again: %v", *drops)
|
||||
}
|
||||
})
|
||||
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
|
||||
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
|
||||
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
|
||||
return func() {
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
|
||||
kv.expire(runBoundPrefix + key)
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
}
|
||||
taken := func(t *testing.T, r *compositionRuns) {
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, kv *runsMem, r *compositionRuns)
|
||||
ended bool
|
||||
bound string // the ask the bound key holds, "" for none
|
||||
untaken string
|
||||
journal string
|
||||
}{
|
||||
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
taken(t, r)
|
||||
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
|
||||
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
rerun(kv, r)()
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
|
||||
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
|
||||
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
|
||||
}
|
||||
}, false, "", "run-1", ""},
|
||||
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
}, false, "run-1", "", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv, r, _, _, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, kv, r)
|
||||
run, _, _, _ := r.read(key)
|
||||
if (run.Ended != nil) != c.ended {
|
||||
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
|
||||
}
|
||||
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
|
||||
if held, _, _, _ := r.boundOf(prefix + key); held != want {
|
||||
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
|
||||
}
|
||||
}
|
||||
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
|
||||
t.Errorf("the journal %v does not say %q", *journal, c.journal)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
|
||||
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
|
||||
// superseded.
|
||||
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
|
||||
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, r *compositionRuns)
|
||||
want []said
|
||||
}{
|
||||
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("b")
|
||||
again.Rerun = true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b")}},
|
||||
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("a2")
|
||||
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), gone("a"), held("a2")}},
|
||||
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.leave("main.3.a", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b"), gone("a")}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, r, saids, _, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, r)
|
||||
if !slices.Equal(*saids, c.want) {
|
||||
t.Errorf("said %v, want %v", *saids, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("taken up at the start", func(t *testing.T) {
|
||||
_, r, _, _, _ := setup(t)
|
||||
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
|
||||
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ended := now
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
|
||||
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"c", "d"} {
|
||||
c := reader(id)
|
||||
c.Ended, c.Rerun = &ended, id == "d"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
|
||||
t.Fatalf("the readers taken up: %+v %v", readers, err)
|
||||
}
|
||||
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
|
||||
t.Errorf("the runs of reader a: %v", keys)
|
||||
}
|
||||
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
|
||||
// `<id>.<the open run's ask>`.
|
||||
ids := make(chan [2]string, 3)
|
||||
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
|
||||
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
},
|
||||
say: func(_ string, body any, id string) error {
|
||||
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
|
||||
return nil
|
||||
}}
|
||||
steps.takeUp(context.Background(), r)
|
||||
got := map[string]string{}
|
||||
for range 3 {
|
||||
select {
|
||||
case e := <-ids:
|
||||
got[e[0]] = e[1]
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends of the take-up: %v", got)
|
||||
}
|
||||
}
|
||||
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
|
||||
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
|
||||
type missingRuns struct {
|
||||
*runsMem
|
||||
hide string
|
||||
}
|
||||
|
||||
func (k *missingRuns) keys() ([]string, error) {
|
||||
all, err := k.runsMem.keys()
|
||||
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
|
||||
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
|
||||
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
|
||||
|
||||
// The tools that call the two verbs.
|
||||
const (
|
||||
compositionsTool = "assigner_compositions"
|
||||
compositionsDropTool = "assigner_compositions_drop"
|
||||
)
|
||||
|
||||
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
|
||||
// machine named only those, and the state's fill against its cap.
|
||||
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
|
||||
continue
|
||||
}
|
||||
run.Key = key
|
||||
out.Runs = append(out.Runs, run)
|
||||
}
|
||||
out.Fill, err = r.fill()
|
||||
return out, err
|
||||
}
|
||||
|
||||
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
|
||||
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
|
||||
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
|
||||
// the readers it ended.
|
||||
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
|
||||
why := strings.TrimSpace(in.Why)
|
||||
if in.Candidate == "" || in.Machine == "" || why == "" {
|
||||
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
|
||||
}
|
||||
key := runKeyOf(in.Candidate, in.Machine)
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, err
|
||||
}
|
||||
if !ok {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
|
||||
}
|
||||
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
|
||||
}
|
||||
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
|
||||
answer := meshwants.CompositionsDropAnswer{Run: run}
|
||||
answer.Run.Key = key
|
||||
if run.Ended == nil {
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
ids := make([]string, 0, len(run.Readers))
|
||||
for _, reader := range run.Readers {
|
||||
ids = append(ids, reader.ID)
|
||||
}
|
||||
c.leave(ids) // their waits say no end of their own
|
||||
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
|
||||
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
|
||||
return answer, err
|
||||
}
|
||||
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
|
||||
for _, reader := range run.Readers {
|
||||
if reader.Ended != nil && !reader.Rerun {
|
||||
continue // it ended passed already and keeps its verdict (A-470r)
|
||||
}
|
||||
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = run.Ask
|
||||
}
|
||||
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
|
||||
answer.Ended = append(answer.Ended, reader)
|
||||
}
|
||||
}
|
||||
// The dropped run's key goes while it holds that run: a run started on it since stands.
|
||||
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
|
||||
r.heard()
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
|
||||
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsInput
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
}
|
||||
return r.compositions(in)
|
||||
},
|
||||
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsDropInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return c.drop(r, record, caller, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
|
||||
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
call := func(verb string, args map[string]any, names ...string) (any, error) {
|
||||
in := map[string]any{}
|
||||
for _, n := range names {
|
||||
if v := arg(args, n); v != "" {
|
||||
in[n] = v
|
||||
}
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
|
||||
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositions, args, "candidate", "machine")
|
||||
}},
|
||||
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
|
||||
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
|
||||
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
|
||||
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
|
||||
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
|
||||
ends := newSaidEvents()
|
||||
var endIDs []string // the message id of each end said
|
||||
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
|
||||
endIDs = append(endIDs, id)
|
||||
return ends.say(event, body, id)
|
||||
}}
|
||||
moves := 0
|
||||
move := func(meshwants.CompositionRun) { moves++ }
|
||||
r.moved.Store(&move)
|
||||
for _, m := range []string{"n1", "n2"} {
|
||||
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"ask-p", "ask-q"} {
|
||||
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
|
||||
c.Ended, c.Rerun = &now, id == "ask-q"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
var recorded []meshhandactsclient.Request
|
||||
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
|
||||
verbs := compositionVerbs(r, c, record)
|
||||
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
|
||||
read, _ := out.(meshwants.CompositionsAnswer)
|
||||
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
|
||||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
|
||||
t.Fatalf("compositions = %+v %v", out, err)
|
||||
}
|
||||
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
|
||||
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
|
||||
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
|
||||
}
|
||||
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
|
||||
dropped, _ := out.(meshwants.CompositionsDropAnswer)
|
||||
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
|
||||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
|
||||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
|
||||
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
|
||||
}
|
||||
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
|
||||
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
|
||||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
|
||||
t.Errorf("the reader's end %+v", end)
|
||||
}
|
||||
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
|
||||
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
|
||||
}
|
||||
// The tools call the verbs with what they were given.
|
||||
var asked []string
|
||||
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
|
||||
})
|
||||
for _, tool := range tools {
|
||||
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
|
||||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
|
||||
t.Errorf("the tools asked %v", asked)
|
||||
}
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
|
||||
// composition step of one module's pipeline that the runner asks, runs it, and says its end as the seat's event
|
||||
// step-ended.<node>, the ask's id as the event's id and the event's `id`, which the runner holds the end to.
|
||||
|
||||
// stepOutcome is what one composition step came to.
|
||||
type stepOutcome struct {
|
||||
Outcome string
|
||||
Said string
|
||||
Warnings []string
|
||||
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
|
||||
Shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// compositionSteps takes the composition steps and says their ends.
|
||||
type compositionSteps struct {
|
||||
node string
|
||||
// say says one event of the seat mesh-wants (wantsPublisher).
|
||||
say publisher
|
||||
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
|
||||
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
// validate runs one validate step and resumeValidate judges one taken up at the start (validatestep.go, design 47 H §745);
|
||||
// nil takes no validate step.
|
||||
validate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
resumeValidate func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
|
||||
// writes none.
|
||||
ended func(id string, run meshwants.StepRun, at time.Time)
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
|
||||
ticks func(d time.Duration) (<-chan time.Time, func())
|
||||
// taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not
|
||||
// pulled yet are kept there too (H-A-404).
|
||||
taken takenStore
|
||||
takenBound time.Duration
|
||||
// inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777).
|
||||
inProgress func() time.Duration
|
||||
mu sync.Mutex
|
||||
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is
|
||||
// dropped (the cause an *askDrop).
|
||||
stops map[string]context.CancelCauseFunc
|
||||
// held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said.
|
||||
held map[string][]queueMsg
|
||||
}
|
||||
|
||||
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
|
||||
const progressEvery = time.Minute
|
||||
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
|
||||
// composition runs it waits on (§723, §729).
|
||||
type stepPhase struct {
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// setShared names the runs the step ctx runs waits on, when it runs one.
|
||||
func setShared(ctx context.Context, shared []meshwants.Shared) {
|
||||
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
||||
p.mu.Lock()
|
||||
p.shared = shared
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *stepPhase) waitsOn() []meshwants.Shared {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.shared
|
||||
}
|
||||
|
||||
type phaseKey struct{}
|
||||
|
||||
// setPhase sets the phase of the step ctx runs, when it runs one.
|
||||
func setPhase(ctx context.Context, phase string) {
|
||||
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
||||
p.mu.Lock()
|
||||
p.phase = phase
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *stepPhase) get() string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.phase
|
||||
}
|
||||
|
||||
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
|
||||
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
|
||||
switch {
|
||||
case step == meshwants.StepRunStepComposition && resumed:
|
||||
return c.resume
|
||||
case step == meshwants.StepRunStepComposition:
|
||||
return c.run
|
||||
case step == meshwants.StepRunStepValidate && resumed:
|
||||
return c.resumeValidate
|
||||
case step == meshwants.StepRunStepValidate:
|
||||
return c.validate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
|
||||
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
|
||||
// says nothing: the runner ended it already.
|
||||
func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
|
||||
started := c.now().UTC()
|
||||
ctx, stopWait := context.WithCancelCause(parent)
|
||||
c.mu.Lock()
|
||||
if c.stops == nil {
|
||||
c.stops = map[string]context.CancelCauseFunc{}
|
||||
}
|
||||
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
|
||||
c.mu.Unlock()
|
||||
stopWait(nil)
|
||||
return
|
||||
}
|
||||
c.stops[id] = stopWait
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
delete(c.stops, id)
|
||||
c.mu.Unlock()
|
||||
stopWait(nil)
|
||||
}()
|
||||
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
|
||||
stop := c.sayProgress(id, run.Asker, phase)
|
||||
stopInProgress := c.inProgressEvery(id)
|
||||
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
stopInProgress()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start
|
||||
return
|
||||
}
|
||||
var d *askDrop
|
||||
if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401)
|
||||
c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again)
|
||||
return
|
||||
}
|
||||
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
|
||||
c.settle(id, true) // the runner ended it already: its ask is over
|
||||
return
|
||||
}
|
||||
c.sayEnd(id, run, &started, o, again)
|
||||
}
|
||||
|
||||
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
|
||||
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
|
||||
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
|
||||
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
|
||||
ended := c.now().UTC()
|
||||
if started == nil {
|
||||
started = &ended
|
||||
}
|
||||
step := meshwants.StepEndedStepComposition
|
||||
if run.Step == meshwants.StepRunStepValidate {
|
||||
step = meshwants.StepEndedStepValidate
|
||||
}
|
||||
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: step, Module: run.Module,
|
||||
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
|
||||
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
|
||||
msgID := id
|
||||
if again != "" {
|
||||
msgID = id + "." + again
|
||||
}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
|
||||
c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err)
|
||||
c.settle(id, false)
|
||||
return
|
||||
}
|
||||
c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777)
|
||||
// One journal line per ask answered, in the one place it is answered (design 47 H2 §792).
|
||||
c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome)
|
||||
if c.ended != nil {
|
||||
c.ended(id, run, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
|
||||
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
|
||||
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
|
||||
return func(run meshwants.CompositionRun) {
|
||||
for _, reader := range run.Readers {
|
||||
if !reader.Rerun || c.running(reader.ID) {
|
||||
continue
|
||||
}
|
||||
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// running is whether the step asked under id runs here.
|
||||
func (c *compositionSteps) running(id string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.stops[id] != nil
|
||||
}
|
||||
|
||||
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
|
||||
func (c *compositionSteps) leave(ids []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
if stop := c.stops[id]; stop != nil {
|
||||
stop(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayProgress says the step's progress on the seat's step-progress.<node> once a minute until the returned stop is called (design
|
||||
// 47 G §496, A-262): a ticker inside the running step, as the build seats' step runner says its own beat.
|
||||
func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop func()) {
|
||||
ticks := c.ticks
|
||||
if ticks == nil {
|
||||
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
|
||||
}
|
||||
tick, stopTicks := ticks(progressEvery)
|
||||
done := make(chan struct{})
|
||||
ended := make(chan struct{})
|
||||
go func() {
|
||||
defer close(ended)
|
||||
defer stopTicks()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick:
|
||||
at := c.now().UTC()
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: asker, Phase: phase.get(), At: at, Shared: phase.waitsOn()}
|
||||
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
c.logf("saying the progress of the composition step %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { close(done); <-ended }
|
||||
}
|
||||
-398
@@ -1,398 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §354, §22, §63, §64, §389, A-240, A-252, A-254: the assigner started whole takes the runner's composition step of a
|
||||
// module placed on three machines, one of them away: it asks the two others' node-engines for their inputs, puts one run ask per
|
||||
// machine on the Go build seat's check queue under its name, its program the node-engine build each machine runs; the stand-in
|
||||
// build seat reads each request by the seat mesh-wants' verb input in chunks and says its step-ended with the composer's verdict;
|
||||
// the step's step-ended fails naming the machine that does not compose, warns of the machine away, and no input stays kept.
|
||||
func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const engineCommit = "1111111111111111111111111111111111111111"
|
||||
// The assigner's durable on EVENTS hears, beside the bed's, the build seat's step-ended and the watchman's machine-away and
|
||||
// machine-back, as the grants part derives them from its consumes; what each node runs it hears on its kept stream (the bed's).
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll, meshwatchdogs.SubjectMachineAwayAll, meshwatchdogs.SubjectMachineBackAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
awaitKept(t, b, nodeengine.StreamRunning, 2)
|
||||
away, _ := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: "n3", At: time.Now().UTC(), LastHeard: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, meshwatchdogs.SubjectMachineAway("n3"), away); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The trunk's catalog half holds the module and the node-engine, the module claiming the seat node-engine.
|
||||
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: engineCommit, Path: ".",
|
||||
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine,
|
||||
"m": {Module: "m", Fp: "src2:m1", Repository: "novox/m", Commit: engineCommit, Path: ".", Manifest: map[string]any{"module": "m", "version": "0.1"}}}
|
||||
b.cat.rev[trunkBranch] = 3
|
||||
b.cat.mu.Unlock()
|
||||
for _, node := range []string{"n1", "n2", "n3"} {
|
||||
b.place(t, node, "m")
|
||||
}
|
||||
// The node-engines of n1 and n2 answer compose-candidate with their inputs; n3 is never asked.
|
||||
asked := make(chan string, 8)
|
||||
engines, err := nc.Subscribe("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) {
|
||||
node := strings.Split(m.Subject, ".")[5]
|
||||
asked <- node
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"candidate": "main.3", "node": node, "verdict": "composed", "input": map[string]any{"node": node},
|
||||
"modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}, "node": node})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = engines.Unsubscribe() })
|
||||
// mesh-catalog's entry answers the node-engine build's entry by its fingerprint.
|
||||
entries, err := nc.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: engine}, "node": "n0"})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = entries.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
// The runner hears the step's end.
|
||||
ended := make(chan meshwants.StepEnded, 1)
|
||||
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data, &e)
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ends.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
|
||||
from := b.journal.mark()
|
||||
b.start(t)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
|
||||
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
|
||||
|
||||
select {
|
||||
case <-b.seatListRead: // the seat server admits a caller of input once the seat list is read (A-278)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
|
||||
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
|
||||
|
||||
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
|
||||
// its end with the composer's verdict: n1 composes, n2 does not.
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
// A-163: the assigner asks under its seat's own name.
|
||||
subject := "mesh.seat.node-build-go.accept.check.mesh-wants"
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
programs := map[string]string{}
|
||||
for range 2 {
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("no run ask is put: %v", err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil {
|
||||
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Machines) != 1 {
|
||||
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
||||
}
|
||||
if len(req.Seats) == 0 { // the revision names no definition: the request carries the trunk seat list's (A-467)
|
||||
t.Errorf("the request of %s carries no seat definition", a.ID)
|
||||
}
|
||||
node := req.Machines[0]
|
||||
programs[node] = a.Run.Program.Commit
|
||||
m := composer.Machine{Composes: true}
|
||||
if node == "n2" {
|
||||
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x"}}
|
||||
}
|
||||
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
|
||||
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(v), At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var e meshwants.StepEnded
|
||||
select {
|
||||
case e = <-ended:
|
||||
case <-b.ctx.Done():
|
||||
t.Fatal("the composition step said no end")
|
||||
}
|
||||
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
|
||||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
|
||||
e.Warnings[1] != "n2: the composer names no culprits" {
|
||||
t.Fatalf("the step ended %+v", e)
|
||||
}
|
||||
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
|
||||
t.Errorf("the composers were built at %v, want the node-engine build each machine runs", programs)
|
||||
}
|
||||
close(asked)
|
||||
for node := range asked {
|
||||
if node == "n3" {
|
||||
t.Error("the machine away was asked for its input")
|
||||
}
|
||||
}
|
||||
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
|
||||
inputs := b.buckets[buckets.Inputs()]
|
||||
w, err := inputs.WatchAll(b.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
keys, _ := inputs.Keys(b.ctx)
|
||||
if len(keys) == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-w.Updates():
|
||||
case <-deadline:
|
||||
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
|
||||
}
|
||||
}
|
||||
queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777)
|
||||
}
|
||||
|
||||
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
|
||||
// chunks on the call's subjects, each acknowledged.
|
||||
func readInput(t *testing.T, nc *nats.Conn, in buildstep.RunInput) []byte {
|
||||
t.Helper()
|
||||
call := "standin" + strings.Repeat("0", 16)
|
||||
chunks := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: call}
|
||||
sub, err := nc.SubscribeSync(chunks.Chunks())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
body, _ := json.Marshal(meshwants.InputInput{Digest: in.Digest, Call: call})
|
||||
// On the call's subject naming its caller, the stand-in build seat's account: the seat server admits no unproven caller (A-179).
|
||||
reply, err := nc.Request("mesh.seat.mesh-wants.call.input.n1~mesh-build-go", body, 10*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, said, ok := buscalls.Unwrap(reply.Data)
|
||||
var a meshwants.InputAnswer
|
||||
if !ok || json.Unmarshal(raw, &a) != nil || a.Chunks < 1 {
|
||||
t.Fatalf("the verb input answered %s: %s", reply.Data, said)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
next := func() (string, []byte, error) {
|
||||
m, err := sub.NextMsg(buscalls.SeatAckWithin)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return m.Subject, m.Data, nil
|
||||
}
|
||||
ack := func(subject string) error { _, err := nc.Request(subject, nil, 10*time.Second); return err }
|
||||
if _, err := chunks.Receive(a.Chunks, next, ack, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := in.Verify(out.Bytes()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// Design 47 H §745 (acceptance test 117, the assigner's half; H-A-108), started whole on the bus: the assigner serves the step
|
||||
// validate on the seat mesh-wants. A take-in whose manifest names an unknown field fails it `manifest-refused` naming it, the
|
||||
// validator being the revision's own entry of the module holding the seat mesh-catalog, run on the Go build seat over the
|
||||
// revision's manifests and the previous revision's; a take-in that changes a manifest with no problem passes it; a take-in whose
|
||||
// entries make a cycle fails it `cycle` naming the cycle, with no validator asked.
|
||||
func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const commit = "2222222222222222222222222222222222222222"
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commit, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
|
||||
entry := func(module, digest string, manifest map[string]any, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
||||
manifest["module"] = module
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: manifest, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.rev[trunkBranch] = 9
|
||||
b.cat.at = map[string]map[int64][]meshcatalog.CatalogEntry{trunkBranch: {
|
||||
6: {indexer, entry("web", "d-web-1", map[string]any{"version": "1"})},
|
||||
7: {indexer, entry("web", "d-web-2", map[string]any{"version": "2", "colour": "red"})},
|
||||
8: {indexer, entry("web", "d-web-3", map[string]any{"version": "3"})},
|
||||
9: {indexer, entry("a", "d-a", map[string]any{}, meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}),
|
||||
entry("b", "d-b", map[string]any{})},
|
||||
}}
|
||||
b.cat.mu.Unlock()
|
||||
ended := make(chan meshwants.StepEnded, 4)
|
||||
ends, err := nc.Subscribe("mesh.seat.mesh-wants.event.step-ended.n1", func(m *nats.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data, &e)
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ends.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
// Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken)
|
||||
// and ended after its start.
|
||||
taken := make(chan *nats.Msg, 8)
|
||||
takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = takens.Unsubscribe() })
|
||||
subscribed(t, nc)
|
||||
putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a",
|
||||
Branch: trunkBranch, Revision: 9, Commit: commit})
|
||||
from := b.journal.mark()
|
||||
b.start(t)
|
||||
b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second)
|
||||
select {
|
||||
case m := <-taken:
|
||||
var at meshwants.AskTakenEvent
|
||||
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") {
|
||||
t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped")
|
||||
}
|
||||
select {
|
||||
case e := <-ended:
|
||||
if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
|
||||
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the validate ask put while the assigner was stopped said no end")
|
||||
}
|
||||
select {
|
||||
case <-b.seatListRead:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants", AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := func(revision int64, module string, run func()) meshwants.StepEnded {
|
||||
t.Helper()
|
||||
id := fmt.Sprintf("validate-%d", revision)
|
||||
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
|
||||
Branch: trunkBranch, Revision: revision, Commit: commit})
|
||||
var asked meshwants.StepRun
|
||||
_ = json.Unmarshal(step, &asked)
|
||||
putStep(b.ctx, t, b.js, id, asked)
|
||||
if run != nil {
|
||||
run()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case e := <-ended:
|
||||
if e.ID == id {
|
||||
return e
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("the validate step of revision %d said no end", revision)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The stand-in Go build seat runs the validator: it refuses a manifest naming the field colour, and accepts every other.
|
||||
validator := func() {
|
||||
t.Helper()
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(30 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("no validator run is asked: %v", err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil || a.Run.Program.Repository != "novox/mesh-delivery" || a.Run.Program.Commit != commit {
|
||||
t.Fatalf("the validator's run ask %s: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase {
|
||||
t.Fatalf("the validator's request: %+v %v", req, err)
|
||||
}
|
||||
// The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443).
|
||||
if list, digest := seatListBytes(); req.SeatList != digest {
|
||||
t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest)
|
||||
} else if got, _ := json.Marshal(req.Seats); string(got) != string(list) {
|
||||
t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list))
|
||||
}
|
||||
v := composer.Verdict{V: composer.Version}
|
||||
for _, c := range req.Cases {
|
||||
reads := map[string]composer.ManifestRead{}
|
||||
for module, m := range c.Manifests {
|
||||
reads[module] = composer.ManifestRead{Accepted: true}
|
||||
if strings.Contains(string(m.Raw), "colour") {
|
||||
reads[module] = composer.ManifestRead{Refused: `unknown field "colour"`}
|
||||
}
|
||||
}
|
||||
v.Cases = append(v.Cases, composer.CaseVerdict{Name: c.Name, Manifests: reads})
|
||||
}
|
||||
out, _ := json.Marshal(v)
|
||||
end, _ := json.Marshal(buildstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: string(out), At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodebuildgo.SubjectStepEnded("b1"), end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if e := ask(7, "web", validator); e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(e.Said, reasonManifestRefused+": ") || !strings.Contains(e.Said, `web: unknown field "colour"`) {
|
||||
t.Errorf("the validate step of a manifest naming an unknown field ended %+v", e)
|
||||
}
|
||||
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
|
||||
}
|
||||
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
|
||||
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
|
||||
}
|
||||
queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777)
|
||||
}
|
||||
@@ -1,510 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled.
|
||||
type fakeMsg struct {
|
||||
data []byte
|
||||
h nats.Header
|
||||
mu sync.Mutex
|
||||
acks, naks, terms, inProgresss int
|
||||
}
|
||||
|
||||
func (m *fakeMsg) Data() []byte { return m.data }
|
||||
func (m *fakeMsg) Headers() nats.Header { return m.h }
|
||||
func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil }
|
||||
func (m *fakeMsg) Ack() error { return m.count(&m.acks) }
|
||||
func (m *fakeMsg) Nak() error { return m.count(&m.naks) }
|
||||
func (m *fakeMsg) Term() error { return m.count(&m.terms) }
|
||||
func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) }
|
||||
func (m *fakeMsg) settled() (acks, naks, terms, progress int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.acks, m.naks, m.terms, m.inProgresss
|
||||
}
|
||||
|
||||
// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set.
|
||||
type fakeTaken struct {
|
||||
mu sync.Mutex
|
||||
values map[string][]byte
|
||||
ttls map[string]time.Duration
|
||||
refuse error
|
||||
}
|
||||
|
||||
func newFakeTaken() *fakeTaken {
|
||||
return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}}
|
||||
}
|
||||
|
||||
func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.refuse != nil {
|
||||
return false, f.refuse
|
||||
}
|
||||
key := seat + "." + ask
|
||||
if _, ok := f.values[key]; ok {
|
||||
return false, nil
|
||||
}
|
||||
f.values[key], f.ttls[key] = []byte(`{}`), bound
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
v, ok := f.values[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: key, Value: v}, nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) {
|
||||
raw, err := json.Marshal(value)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.values[key], f.ttls[key] = raw, ttl
|
||||
return 1, err
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Delete(key string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeTaken) Keys() ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none.
|
||||
func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg {
|
||||
if c.taken == nil {
|
||||
c.taken = newFakeTaken()
|
||||
}
|
||||
m := &fakeMsg{data: body, h: nats.Header{}}
|
||||
if id != "" {
|
||||
m.h.Set("x-event-id", id)
|
||||
}
|
||||
c.pulled(context.Background(), m)
|
||||
return m
|
||||
}
|
||||
|
||||
// saidEvents keeps what a publisher says.
|
||||
type saidEvents struct {
|
||||
mu sync.Mutex
|
||||
said map[string]any
|
||||
ids map[string]string
|
||||
heard chan string
|
||||
}
|
||||
|
||||
func newSaidEvents() *saidEvents {
|
||||
return &saidEvents{said: map[string]any{}, ids: map[string]string{}, heard: make(chan string, 8)}
|
||||
}
|
||||
|
||||
func (s *saidEvents) say(event string, body any, id string) error {
|
||||
s.mu.Lock()
|
||||
s.said[event], s.ids[event] = body, id
|
||||
s.mu.Unlock()
|
||||
s.heard <- event
|
||||
return nil
|
||||
}
|
||||
|
||||
// AS5 (§22, §354): the assigner takes mesh-wants' accepted kind `step` for the composition step and says the seat's
|
||||
// step-ended.<node> with the ask's id, what the step was asked at and what it came to; an ask of another step is left.
|
||||
func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
said := newSaidEvents()
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
var asked meshwants.StepRun
|
||||
var lines []string
|
||||
var linesMu sync.Mutex
|
||||
logf := func(f string, a ...any) {
|
||||
linesMu.Lock()
|
||||
lines = append(lines, fmt.Sprintf(f, a...))
|
||||
linesMu.Unlock()
|
||||
}
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf,
|
||||
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
|
||||
asked = run
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
|
||||
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
|
||||
}}
|
||||
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
|
||||
if m := pullOne(c, "ask-0", other); m.terms != 1 {
|
||||
t.Fatal("a step ask of another step is taken")
|
||||
}
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
|
||||
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
|
||||
if m := pullOne(c, "", body); m.terms != 1 {
|
||||
t.Fatal("a step ask with no id is taken")
|
||||
}
|
||||
if m := pullOne(c, "ask-1", body); m.terms != 0 {
|
||||
t.Fatal("the composition step is not taken")
|
||||
}
|
||||
if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777)
|
||||
t.Fatalf("said %s first, want ask-taken", e)
|
||||
}
|
||||
select {
|
||||
case <-said.heard:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no step-ended is said")
|
||||
}
|
||||
event := meshwants.EventStepEnded + ".n1"
|
||||
end, ok := said.said[event].(meshwants.StepEnded)
|
||||
if !ok || said.ids[event] != "ask-1" {
|
||||
t.Fatalf("said %v under %v, want step-ended.n1 under the ask's id", said.said, said.ids)
|
||||
}
|
||||
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
|
||||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
|
||||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
|
||||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
|
||||
t.Fatalf("step-ended %+v", end)
|
||||
}
|
||||
// Design 47 H2 §792: the ask answered is one journal line, said where its end is said.
|
||||
linesMu.Lock()
|
||||
answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") })
|
||||
linesMu.Unlock()
|
||||
if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) {
|
||||
t.Errorf("the journal says %q, want one answered line", answered)
|
||||
}
|
||||
// Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at
|
||||
// the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop.
|
||||
t.Run("work queue", queueCases)
|
||||
}
|
||||
|
||||
func queueCases(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4})
|
||||
type bed struct {
|
||||
c *compositionSteps
|
||||
said *saidEvents
|
||||
taken *fakeTaken
|
||||
runs chan string
|
||||
release chan struct{}
|
||||
tick chan time.Time
|
||||
}
|
||||
newBed := func() *bed {
|
||||
b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)}
|
||||
b.said.heard = make(chan string, 16)
|
||||
run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome {
|
||||
b.runs <- id
|
||||
select {
|
||||
case <-b.release:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}
|
||||
b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute,
|
||||
validate: run, inProgress: func() time.Duration { return 20 * time.Second },
|
||||
ticks: func(d time.Duration) (<-chan time.Time, func()) {
|
||||
if d == 20*time.Second {
|
||||
return b.tick, func() {}
|
||||
}
|
||||
return make(chan time.Time), func() {}
|
||||
}}
|
||||
return b
|
||||
}
|
||||
ended := func(t *testing.T, b *bed) meshwants.StepEnded {
|
||||
t.Helper()
|
||||
for {
|
||||
select {
|
||||
case e := <-b.said.heard:
|
||||
if e == meshwants.EventStepEnded+".n1" {
|
||||
b.said.mu.Lock()
|
||||
defer b.said.mu.Unlock()
|
||||
return b.said.said[e].(meshwants.StepEnded)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no step-ended is said")
|
||||
}
|
||||
}
|
||||
}
|
||||
settledAt := func(t *testing.T, m *fakeMsg, acks int) {
|
||||
t.Helper()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
if a, _, _, _ := m.settled(); a == acks {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks)
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) {
|
||||
b := newBed()
|
||||
m := pullOne(b.c, "ask-1", body)
|
||||
if id := <-b.runs; id != "ask-1" {
|
||||
t.Fatalf("ran %s", id)
|
||||
}
|
||||
b.said.mu.Lock()
|
||||
taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent)
|
||||
takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"]
|
||||
b.said.mu.Unlock()
|
||||
if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute {
|
||||
t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls)
|
||||
}
|
||||
b.tick <- at
|
||||
b.tick <- at
|
||||
// handed again while it runs: held for the run, which answers both at its one end
|
||||
again := pullOne(b.c, "ask-1", body)
|
||||
if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 {
|
||||
t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p)
|
||||
}
|
||||
close(b.release)
|
||||
if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
settledAt(t, again, 1)
|
||||
select {
|
||||
case id := <-b.runs:
|
||||
t.Fatalf("the ask handed again ran again (%s)", id)
|
||||
default:
|
||||
}
|
||||
})
|
||||
t.Run("taken before is run again, said no ask-taken", func(t *testing.T) {
|
||||
b := newBed()
|
||||
_, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute)
|
||||
close(b.release)
|
||||
m := pullOne(b.c, "ask-2", body)
|
||||
<-b.runs
|
||||
ended(t, b)
|
||||
settledAt(t, m, 1)
|
||||
b.said.mu.Lock()
|
||||
_, said := b.said.said[meshwants.EventAskTaken+".ask-2"]
|
||||
b.said.mu.Unlock()
|
||||
if said {
|
||||
t.Fatal("ask-taken is said again for an ask taken before")
|
||||
}
|
||||
})
|
||||
t.Run("a refused take is handed again", func(t *testing.T) {
|
||||
b := newBed()
|
||||
b.taken.refuse = errors.New("no bus")
|
||||
m := pullOne(b.c, "ask-3", body)
|
||||
if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 {
|
||||
t.Fatalf("nakked %d, ran %d", n, len(b.runs))
|
||||
}
|
||||
})
|
||||
t.Run("dropped while it runs", func(t *testing.T) {
|
||||
b := newBed()
|
||||
m := pullOne(b.c, "ask-4", body)
|
||||
<-b.runs
|
||||
if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil {
|
||||
t.Fatal("a drop with no why is taken")
|
||||
}
|
||||
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"})
|
||||
if err != nil || a["held"] != true {
|
||||
t.Fatalf("drop answered %v, %v", a, err)
|
||||
}
|
||||
e := ended(t, b)
|
||||
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
})
|
||||
t.Run("dropped before it was pulled", func(t *testing.T) {
|
||||
b := newBed()
|
||||
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"})
|
||||
if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute {
|
||||
t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls)
|
||||
}
|
||||
m := pullOne(b.c, "ask-5", body)
|
||||
e := ended(t, b)
|
||||
if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" {
|
||||
t.Fatalf("step-ended %+v", e)
|
||||
}
|
||||
settledAt(t, m, 1)
|
||||
if len(b.runs) != 0 {
|
||||
t.Fatal("a dropped ask ran")
|
||||
}
|
||||
if v, _ := b.taken.Get("dropped.ask-5"); v != nil {
|
||||
t.Fatal("the drop's key stays once its ask ended")
|
||||
}
|
||||
})
|
||||
t.Run("the asks tools", func(t *testing.T) {
|
||||
taken := newFakeTaken()
|
||||
_, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute)
|
||||
_, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute)
|
||||
waiting := func(seat string) (stdio.Waiting, error) {
|
||||
if seat != meshwants.Seat {
|
||||
t.Errorf("the waiting asks of %s are read", seat)
|
||||
}
|
||||
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3},
|
||||
{ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil
|
||||
}
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
if strings.HasSuffix(key, "."+meshwants.VerbDrop) {
|
||||
return json.RawMessage(`{"held":false}`), nil
|
||||
}
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
tools := map[string]stdio.Tool{}
|
||||
for _, tool := range asksTools(waiting, taken, ask) {
|
||||
tools[tool.Name] = tool
|
||||
}
|
||||
out, err := tools[asksTool].Run(map[string]any{})
|
||||
a, _ := out.(map[string]any)
|
||||
w, _ := a["waiting"].([]map[string]any)
|
||||
d, _ := a["dropped"].([]map[string]any)
|
||||
if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 ||
|
||||
d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false {
|
||||
t.Fatalf("assigner_asks answered %+v, %v", out, err)
|
||||
}
|
||||
if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 {
|
||||
t.Fatalf("a drop with no why: %v, asked %v", err, asked)
|
||||
}
|
||||
out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"})
|
||||
if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") ||
|
||||
!strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) {
|
||||
t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
|
||||
// asker, the phase the run is in and the time, and says none once it has ended.
|
||||
func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
said := newSaidEvents()
|
||||
tick := make(chan time.Time)
|
||||
stopped := make(chan struct{})
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
release := make(chan struct{})
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
|
||||
ticks: func(d time.Duration) (<-chan time.Time, func()) {
|
||||
if d != time.Minute {
|
||||
t.Errorf("the progress ticks every %s, want a minute", d)
|
||||
}
|
||||
return tick, func() { close(stopped) }
|
||||
},
|
||||
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
|
||||
<-release
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
|
||||
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
|
||||
if m := pullOne(c, "ask-1", body); m.terms != 0 {
|
||||
t.Fatal("the composition step is not taken")
|
||||
}
|
||||
<-said.heard // ask-taken, at the pull (H2 §777)
|
||||
progress := meshwants.EventStepProgress + ".n1"
|
||||
for range 2 {
|
||||
tick <- at
|
||||
if e := <-said.heard; e != progress {
|
||||
t.Fatalf("said %s on a tick, want %s", e, progress)
|
||||
}
|
||||
}
|
||||
said.mu.Lock()
|
||||
p, ok := said.said[progress].(meshwants.StepProgressEvent)
|
||||
said.mu.Unlock()
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
|
||||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
|
||||
t.Fatalf("step-progress %+v", p)
|
||||
}
|
||||
close(release)
|
||||
if e := <-said.heard; e != meshwants.EventStepEnded+".n1" {
|
||||
t.Fatalf("said %s at the end, want step-ended", e)
|
||||
}
|
||||
select {
|
||||
case <-stopped:
|
||||
default:
|
||||
t.Fatal("the ticker is not stopped when the step has ended")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
|
||||
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
|
||||
// reader is left", so a later step starts a new run.
|
||||
func TestASupersededStepLeavesItsRuns(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
step := func(pipeline string) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
|
||||
}
|
||||
said := newSaidEvents()
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
|
||||
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
|
||||
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
r.await(ctx, []string{j.key})
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
|
||||
for _, p := range []string{"main.3.a", "main.3.b"} {
|
||||
if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 {
|
||||
t.Fatal("not taken")
|
||||
}
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
for {
|
||||
wake := r.wake()
|
||||
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
|
||||
break
|
||||
}
|
||||
<-wake
|
||||
}
|
||||
ids, err := r.leave("main.3.a", c.leave)
|
||||
run, _, _, _ := r.read(key)
|
||||
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
|
||||
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
|
||||
}
|
||||
_, err = r.leave("main.3.b", c.leave)
|
||||
run, _, _, _ = r.read(key)
|
||||
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
|
||||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
|
||||
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
n := len(c.stops)
|
||||
c.mu.Unlock()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("the steps that left still wait")
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
said.mu.Lock()
|
||||
defer said.mu.Unlock()
|
||||
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
|
||||
t.Errorf("a step that left says its end: %v", said.said)
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// engineWords is one node's key of the assigner's state engine-reads (design 47 H-A-104): the words its node-engine said it
|
||||
// reads in the last manifest-reads-changed the assigner heard of it, and when the engine said them.
|
||||
type engineWords struct {
|
||||
Words []string `json:"words"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// engineReadsKept is what the assigner holds of its state engine-reads (design 47 H-A-104, §772): written when it hears the
|
||||
// node-engine seat's manifest-reads-changed, and read back from the state's watch at the start, so a restart loses no node's
|
||||
// words; no verb is asked and no node's record is read in its place.
|
||||
type engineReadsKept struct {
|
||||
mu sync.Mutex
|
||||
by map[string]engineWords
|
||||
// put writes one node's key; nil keeps the words in memory only (the state could not be opened).
|
||||
put func(node string, value []byte) error
|
||||
// changed wakes a pass when a node's words changed; nil wakes nothing.
|
||||
changed func()
|
||||
}
|
||||
|
||||
// heard keeps what one manifest-reads-changed says of its node: written into the state first, then held.
|
||||
func (k *engineReadsKept) heard(e nodeengine.ManifestReadsChangedEvent) error {
|
||||
at, err := time.Parse(time.RFC3339Nano, e.At)
|
||||
if err != nil {
|
||||
return fmt.Errorf("at %q is no RFC 3339 time", e.At)
|
||||
}
|
||||
v := engineWords{Words: slices.Clone(e.ManifestReads), At: at}
|
||||
if k.put != nil {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.put(e.Node, raw); err != nil {
|
||||
return fmt.Errorf("the words of %s are not written into the state engine-reads: %w", e.Node, err)
|
||||
}
|
||||
}
|
||||
k.hold(e.Node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// take takes in one key of the state as its watch hands it: at the start every node's words, later the assigner's own writes
|
||||
// (which change nothing) and a key dropped.
|
||||
func (k *engineReadsKept) take(node string, raw []byte, deleted bool) error {
|
||||
if deleted {
|
||||
k.hold(node, engineWords{}, true)
|
||||
return nil
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return fmt.Errorf("the key %s of the state engine-reads does not read: %w", node, err)
|
||||
}
|
||||
k.hold(node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hold sets one node's words, waking a pass when they changed.
|
||||
func (k *engineReadsKept) hold(node string, v engineWords, deleted bool) {
|
||||
k.mu.Lock()
|
||||
old, had := k.by[node]
|
||||
switch {
|
||||
case deleted:
|
||||
delete(k.by, node)
|
||||
default:
|
||||
if k.by == nil {
|
||||
k.by = map[string]engineWords{}
|
||||
}
|
||||
k.by[node] = v
|
||||
}
|
||||
changed := (had && deleted) || (!deleted && (!had || !old.At.Equal(v.At) || !slices.Equal(old.Words, v.Words)))
|
||||
k.mu.Unlock()
|
||||
if changed && k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
}
|
||||
|
||||
// words is what node's engine said it reads, and whether the assigner holds its key; nil k holds none.
|
||||
func (k *engineReadsKept) words(node string) ([]string, bool) {
|
||||
if k == nil {
|
||||
return nil, false
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
v, ok := k.by[node]
|
||||
return slices.Clone(v.Words), ok
|
||||
}
|
||||
|
||||
// nodes is every node whose key the assigner holds, sorted.
|
||||
func (k *engineReadsKept) nodes() []string {
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(k.by))
|
||||
}
|
||||
|
||||
// engineReadsTool lists the assigner's state engine-reads: what each node's node-engine said it reads, and when.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
|
||||
// readEngineReads is the state engine-reads, every node's key or node's alone.
|
||||
func readEngineReads(st store, node string) (any, error) {
|
||||
keys := []string{node}
|
||||
if node == "" {
|
||||
var err error
|
||||
if keys, err = st.keys(engineReadsState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(keys)
|
||||
}
|
||||
out := map[string]engineWords{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := st.get(engineReadsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
if node != "" {
|
||||
return nil, fmt.Errorf("the assigner holds no words of %s: its node-engine said no %s it heard", node, nodeengine.EventManifestReadsChanged)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("the key %s does not read", key), err)
|
||||
}
|
||||
out[key] = v
|
||||
}
|
||||
return map[string]any{"nodes": out}, nil
|
||||
}
|
||||
|
||||
// engineReadsTools is the tool over the runtime's state.
|
||||
func engineReadsTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: engineReadsTool, Description: "What each node's node-engine says it reads of a manifest, as the " +
|
||||
"assigner keeps it in its state engine-reads (design 47 H-A-104): per node the words of the last manifest-reads-changed " +
|
||||
"it heard and when the engine said them. A node with no key states nothing the assigner heard, so its wants are held " +
|
||||
"awaiting its hand placement (§772). With node, that node's key alone.",
|
||||
Input: schema(map[string]string{"node": "one node; every node when left out"}, nil),
|
||||
Run: func(a map[string]any) (any, error) { return readEngineReads(st, arg(a, "node")) }}}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// runtimeObjects is the kept data assignment-revisions as the bundle reaches it through its runtime (the old objects
|
||||
// section of a foundation part, design 47 lines 1362-1366).
|
||||
type runtimeObjects struct{ o busstate.Objects }
|
||||
|
||||
func (r runtimeObjects) Put(_ context.Context, name string, data []byte) error {
|
||||
_, err := r.o.Put(name, data, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func (r runtimeObjects) Get(_ context.Context, name string) ([]byte, error) {
|
||||
d, err := r.o.Get(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.Info.Name == "" || d.Info.Deleted {
|
||||
return nil, assigngraph.ErrNoObject
|
||||
}
|
||||
return d.Data, nil
|
||||
}
|
||||
|
||||
func (r runtimeObjects) Names(context.Context) ([]string, error) {
|
||||
infos, err := r.o.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := []string{}
|
||||
for _, i := range infos {
|
||||
if !i.Deleted {
|
||||
names = append(names, i.Name)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// graphVerbs are the seat mesh-wants' verbs revision, entry and difference (design 47 lines 1365-1366: an assignment graph
|
||||
// revision is served by the seat's verbs), each reading only the kept revisions.
|
||||
func graphVerbs(o assigngraph.Objects) map[string]func(map[string]any) (any, error) {
|
||||
return map[string]func(map[string]any) (any, error){
|
||||
meshwants.VerbRevision: func(a map[string]any) (any, error) {
|
||||
var in meshwants.RevisionInput
|
||||
if err := decodeArgs(a, &in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return assigngraph.Revision(context.Background(), o, in)
|
||||
},
|
||||
meshwants.VerbEntry: func(a map[string]any) (any, error) {
|
||||
var in meshwants.EntryInput
|
||||
if err := decodeArgs(a, &in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return assigngraph.Entry(context.Background(), o, in)
|
||||
},
|
||||
meshwants.VerbDifference: func(a map[string]any) (any, error) {
|
||||
var in meshwants.DifferenceInput
|
||||
if err := decodeArgs(a, &in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return assigngraph.Difference(context.Background(), o, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// decodeArgs reads a verb's arguments into its input type; a caller's own fields beside them are the runtime's.
|
||||
func decodeArgs(a map[string]any, into any) error {
|
||||
raw, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, into)
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
|
||||
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
|
||||
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
|
||||
|
||||
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
|
||||
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
|
||||
return fillIn(ctx, g.objects)
|
||||
}
|
||||
|
||||
// passObjects is the store as the pass reads its fill: a handle of its own where one is given, so the pass's read never shares one
|
||||
// with a keeping that runs beside it (design 47 H-A-261).
|
||||
func (g *graphKeeper) passObjects() assigngraph.Objects {
|
||||
if g.reads != nil {
|
||||
return g.reads
|
||||
}
|
||||
return g.objects
|
||||
}
|
||||
|
||||
// fillIn is the fill of the bucket o, when it says one and has a cap.
|
||||
func fillIn(ctx context.Context, o assigngraph.Objects) (used, capacity int64, ok bool) {
|
||||
f, is := o.(assigngraph.Filler)
|
||||
if !is {
|
||||
return 0, 0, false
|
||||
}
|
||||
used, capacity, err := f.Fill(ctx)
|
||||
return used, capacity, err == nil && capacity > 0
|
||||
}
|
||||
|
||||
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
|
||||
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
|
||||
// its cap), so a fill reached while the process was down is said too.
|
||||
func (p *passer) fillAtStart(ctx context.Context) {
|
||||
g := p.graph
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case used >= capacity:
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
|
||||
case used*5 >= capacity*4:
|
||||
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
|
||||
}
|
||||
}
|
||||
|
||||
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
|
||||
// 80 % again. It answers true while the store is full.
|
||||
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
|
||||
g := p.graph
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if g.isFull() && used*5 < capacity*4 {
|
||||
g.setFull(false)
|
||||
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
if used+assigngraph.NewBytes(ctx, g.passObjects(), c) <= capacity {
|
||||
g.setFull(false)
|
||||
return false
|
||||
}
|
||||
if !g.isFull() {
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
|
||||
return true
|
||||
}
|
||||
|
||||
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
|
||||
func (p *passer) fillingAfter(ctx context.Context, before int64) {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
|
||||
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
|
||||
type filled struct {
|
||||
objs map[string][]byte
|
||||
base int64
|
||||
cap int64
|
||||
}
|
||||
|
||||
func (f *filled) Put(_ context.Context, name string, data []byte) error {
|
||||
f.objs[name] = data
|
||||
return nil
|
||||
}
|
||||
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
|
||||
if d, ok := f.objs[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, assigngraph.ErrNoObject
|
||||
}
|
||||
func (f *filled) Names(context.Context) ([]string, error) {
|
||||
out := []string{}
|
||||
for n := range f.objs {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *filled) Fill(context.Context) (int64, int64, error) {
|
||||
used := f.base
|
||||
for _, d := range f.objs {
|
||||
used += int64(len(d))
|
||||
}
|
||||
return used, f.cap, nil
|
||||
}
|
||||
|
||||
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
|
||||
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
|
||||
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
p := newPasser(kvSet{}, time.Now())
|
||||
var events []string
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
if _, ok := body.(meshwants.Fill); ok {
|
||||
events = append(events, event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
|
||||
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
|
||||
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
o.base, o.cap = 8500, 10000 // past 80 %, with room for the next revision, which the pass judges itself (H-A-261)
|
||||
p.graph.started = false
|
||||
p.keepGraph(ctx) // a start again: its first pass judges the fill
|
||||
if len(events) != 1 || events[0] != meshwants.EventFilling {
|
||||
t.Fatalf("at the start past 80 %%: %v", events)
|
||||
}
|
||||
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
|
||||
o.base, o.cap = 990, 1000
|
||||
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
|
||||
t.Fatalf("a revision that does not fit: events %v", events)
|
||||
}
|
||||
p.fullBefore(ctx, c)
|
||||
if n := len(events); n != 2 {
|
||||
t.Errorf("full is said once: %v", events)
|
||||
}
|
||||
// H-A-261: the verdict is the pass's own: with the store full the pass writes no want, though a revision is kept.
|
||||
if held := p.keepGraph(ctx); held != heldFull {
|
||||
t.Errorf("a pass with the store full answers %q, want %q", held, heldFull)
|
||||
}
|
||||
o.base = 100
|
||||
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
|
||||
t.Errorf("room: events %v", events)
|
||||
}
|
||||
if held := p.keepGraph(ctx); held != "" {
|
||||
t.Errorf("a pass with room and a revision kept answers %q, want the wants written", held)
|
||||
}
|
||||
}
|
||||
@@ -1,512 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
|
||||
// from the catalog halves it follows, the placements, the first branch of each registration and each node's capability reading,
|
||||
// keeps it in the objects assignment-revisions and says the seat mesh-wants' event assignment-revision; the wants of the pass
|
||||
// that follows are written from the revision it holds.
|
||||
type graphKeeper struct {
|
||||
objects assigngraph.Objects
|
||||
// reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects.
|
||||
reads assigngraph.Objects
|
||||
// halves is every followed branch's catalog graph revision held now.
|
||||
halves func() map[string]mcatc.Half
|
||||
// mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259).
|
||||
mu sync.Mutex
|
||||
// handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false
|
||||
// until one is, and again when its keeping failed or found the store full, so the next pass hands it again.
|
||||
handedRev meshwants.AssignmentRevision
|
||||
handed bool
|
||||
// handedOnce says a revision was handed: the first carries the start's removal (H-A-261).
|
||||
handedOnce bool
|
||||
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
|
||||
kept meshwants.AssignmentRevision
|
||||
source int64
|
||||
ok bool
|
||||
said string
|
||||
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
|
||||
full, started bool
|
||||
// revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision.
|
||||
revisions *revisionRemover
|
||||
// computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one.
|
||||
computed assigngraph.Computed
|
||||
hasComputed bool
|
||||
// keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow).
|
||||
keepMu sync.Mutex
|
||||
// The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at
|
||||
// once, in the caller), and wake makes a pass once a keeping changed what the pass reads.
|
||||
need *keepNeed
|
||||
keeping bool
|
||||
background func(func())
|
||||
wake func()
|
||||
// A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures
|
||||
// counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)).
|
||||
retry time.Duration
|
||||
after func(d time.Duration, f func())
|
||||
failures int
|
||||
}
|
||||
|
||||
// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline).
|
||||
const (
|
||||
keepingRetrySetting = "keeping-retry"
|
||||
defaultKeepingRetry = 30 * time.Second
|
||||
kindKeepingFailed = "keeping-failed"
|
||||
// keepingFailedAt is the failures in a row that raise keeping-failed.
|
||||
keepingFailedAt = 3
|
||||
)
|
||||
|
||||
// keepingRetry reads the setting keeping-retry (§781).
|
||||
func keepingRetry() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged.
|
||||
type keepNeed struct {
|
||||
c *assigngraph.Computed
|
||||
judge bool
|
||||
}
|
||||
|
||||
// keptNow is the revision the wants follow, with its trunk source, and whether one is kept.
|
||||
func (g *graphKeeper) keptNow() (meshwants.AssignmentRevision, int64, bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.kept, g.source, g.ok
|
||||
}
|
||||
|
||||
// isFull says the store did not hold the last revision handed over, so no want is written (design 47 G §181).
|
||||
func (g *graphKeeper) isFull() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.full
|
||||
}
|
||||
|
||||
func (g *graphKeeper) setFull(full bool) {
|
||||
g.mu.Lock()
|
||||
g.full = full
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
// heldNoRevision is the pass's words while no assignment graph revision is kept: it writes no want (design 47 H-A-261).
|
||||
const heldNoRevision = "held: no assignment graph revision is kept yet, so no want is written until the first keeping ends (H-A-261)"
|
||||
|
||||
// heldFull is the pass's words while the next revision does not fit its store (design 47 G §181).
|
||||
const heldFull = "the assignment graph store is full: no want is written (design 47 G §181)"
|
||||
|
||||
// keepGraph computes the revision of this pass, judges the store's fill itself, and hands the revision to the keeping when the
|
||||
// pass's inputs differ from the last handed. The keeping and the removing of revisions run off the pass loop, one at a time, and the
|
||||
// pass never waits on them: it judges and writes the wants from the graph it holds (design 47 H-A-259). It answers why the pass
|
||||
// writes no want, or "" when it writes them: none while the next revision does not fit the store (G §181), and none while no
|
||||
// revision is kept (H-A-261), whose keeping wakes a pass when it ends.
|
||||
func (p *passer) keepGraph(ctx context.Context) string {
|
||||
g := p.graph
|
||||
if g == nil || g.objects == nil || g.halves == nil {
|
||||
return ""
|
||||
}
|
||||
c := assigngraph.Compute(p.graphInputs())
|
||||
g.mu.Lock()
|
||||
g.computed, g.hasComputed = c, true
|
||||
g.mu.Unlock()
|
||||
for _, u := range c.Uncomputed {
|
||||
if g.said != u {
|
||||
g.said = u
|
||||
p.logf("the assignment graph leaves out %s", u)
|
||||
}
|
||||
}
|
||||
if !g.started { // the first pass judges the fill once (graphfill.go)
|
||||
g.started = true
|
||||
p.fillAtStart(ctx)
|
||||
}
|
||||
if p.fullBefore(ctx, c) {
|
||||
g.mu.Lock()
|
||||
g.handed = false // handed once it fits
|
||||
g.mu.Unlock()
|
||||
return heldFull
|
||||
}
|
||||
g.mu.Lock()
|
||||
same := g.handed && assigngraph.Same(g.handedRev, c.Revision)
|
||||
first := !g.handedOnce
|
||||
if !same {
|
||||
g.handedRev, g.handed, g.handedOnce = c.Revision, true, true
|
||||
}
|
||||
g.mu.Unlock()
|
||||
if !same {
|
||||
// The start's removal of the revisions not current folds into the first keeping: it keeps the first pass's revision, then
|
||||
// removes (H-A-261).
|
||||
p.needKeeping(ctx, keepNeed{c: &c, judge: first})
|
||||
}
|
||||
if _, _, ok := g.keptNow(); !ok {
|
||||
return heldNoRevision
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// needKeeping hands a need to the keeping: one keeping runs at a time, and a need that comes while one runs is folded into the one
|
||||
// that waits (the newest revision, and a judging asked by either), so it runs once more (design 47 H-A-259).
|
||||
func (p *passer) needKeeping(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
if w := g.need; w != nil {
|
||||
n.judge = n.judge || w.judge
|
||||
if n.c == nil {
|
||||
n.c = w.c
|
||||
}
|
||||
}
|
||||
g.need = &n
|
||||
if g.keeping {
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.keeping = true
|
||||
g.mu.Unlock()
|
||||
run := func() {
|
||||
for {
|
||||
g.mu.Lock()
|
||||
next := g.need
|
||||
g.need = nil
|
||||
if next == nil {
|
||||
g.keeping = false
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.mu.Unlock()
|
||||
p.keepOne(ctx, *next)
|
||||
}
|
||||
}
|
||||
if g.background != nil {
|
||||
g.background(run)
|
||||
return
|
||||
}
|
||||
run()
|
||||
}
|
||||
|
||||
// keepOne keeps one computed revision, says assignment-revision for a revision kept now and removes the
|
||||
// revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads.
|
||||
func (p *passer) keepOne(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
g.keepMu.Lock()
|
||||
defer g.keepMu.Unlock()
|
||||
start := time.Now()
|
||||
changed := false
|
||||
defer func() {
|
||||
p.logf("the keeping of the assignment graph took %s", time.Since(start).Round(time.Millisecond))
|
||||
if changed && g.background != nil && g.wake != nil {
|
||||
g.wake()
|
||||
}
|
||||
}()
|
||||
if n.c != nil {
|
||||
_, _, had := g.keptNow()
|
||||
fresh := p.keepRevision(ctx, *n.c)
|
||||
_, _, has := g.keptNow()
|
||||
n.judge = n.judge || fresh
|
||||
changed = fresh || had != has // the first revision kept: the pass writes the wants it held (H-A-261)
|
||||
}
|
||||
if n.judge && g.revisions != nil {
|
||||
g.revisions.judge() // the revision before ceased to be current
|
||||
}
|
||||
}
|
||||
|
||||
// keepRevision keeps c when the store holds it, and says whether a revision was kept now.
|
||||
func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool {
|
||||
fresh, _ := p.keepRevisionOf(ctx, c)
|
||||
return fresh
|
||||
}
|
||||
|
||||
// keepRevisionOf is keepRevision answering the fault the keeping met too.
|
||||
func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) {
|
||||
g := p.graph
|
||||
again := func() { // the next pass hands this revision again
|
||||
g.mu.Lock()
|
||||
g.handed = false
|
||||
g.mu.Unlock()
|
||||
}
|
||||
used, _, _ := g.fillOf(ctx)
|
||||
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
|
||||
if err != nil {
|
||||
again()
|
||||
p.keepingFailed(ctx, c, err)
|
||||
return false, err
|
||||
}
|
||||
p.keepingSucceeded()
|
||||
var source int64
|
||||
for _, cat := range rev.Catalogs {
|
||||
if cat.Branch == p.w.trunkName() {
|
||||
source = cat.Source
|
||||
}
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.kept, g.ok, g.source = rev, true, source
|
||||
g.mu.Unlock()
|
||||
if !fresh {
|
||||
return false, nil
|
||||
}
|
||||
p.fillingAfter(ctx, used)
|
||||
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
|
||||
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
|
||||
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed
|
||||
// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed
|
||||
// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls.
|
||||
func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) {
|
||||
g := p.graph
|
||||
if strings.TrimSpace(in.Why) == "" {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now")
|
||||
}
|
||||
if g == nil || g.objects == nil {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here")
|
||||
}
|
||||
g.mu.Lock()
|
||||
c, ok := g.computed, g.hasComputed
|
||||
g.mu.Unlock()
|
||||
if !ok {
|
||||
return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet")
|
||||
}
|
||||
g.keepMu.Lock()
|
||||
defer g.keepMu.Unlock()
|
||||
p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why)
|
||||
if _, err := p.keepRevisionOf(ctx, c); err != nil {
|
||||
return meshwants.KeepNowAnswer{Fault: err.Error()}, nil
|
||||
}
|
||||
rev, _, _ := g.keptNow()
|
||||
return meshwants.KeepNowAnswer{Revision: &rev}, nil
|
||||
}
|
||||
|
||||
// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now.
|
||||
func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.KeepNowInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return p.keepNow(ctx, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
|
||||
// node's capability reading and the first branch of each registration.
|
||||
func (p *passer) graphInputs() assigngraph.Inputs {
|
||||
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
|
||||
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
|
||||
// (design 47 G §708).
|
||||
if r := p.seats.Reader(); r != nil {
|
||||
in.Definitions = r.Definitions()
|
||||
}
|
||||
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
|
||||
first := p.w.trunkName()
|
||||
in.FirstBranch = func(string) string { return first }
|
||||
if p.ends != nil {
|
||||
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// capabilityDigests is each node's capability reading, by node: the statement its engine made of what it reads, and the
|
||||
// capabilities its tool runner states with the fingerprint of the runner that announced them, as the node's record carries
|
||||
// them (design 46 approval mark §6, lead answer A-203), the digest over all of it.
|
||||
func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem {
|
||||
records := p.w.records()
|
||||
known := p.w.knownNodes()
|
||||
slices.Sort(known)
|
||||
var out []meshwants.AssignmentRevisionCapabilitiesItem
|
||||
for _, node := range known {
|
||||
reads, read := p.reads.words(node)
|
||||
rec, recorded := records[node]
|
||||
if !read && !recorded {
|
||||
continue
|
||||
}
|
||||
slices.Sort(reads)
|
||||
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
|
||||
Fingerprint: rec.RunnerFingerprint}
|
||||
slices.Sort(item.Readings)
|
||||
raw, _ := json.Marshal(struct {
|
||||
Reads []string `json:"reads"`
|
||||
Readings []string `json:"readings"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
}{reads, item.Readings, item.Fingerprint})
|
||||
sum := sha256.Sum256(raw)
|
||||
item.Digest = "sha256:" + hex.EncodeToString(sum[:])
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sayWanted says `wanted` for a want written or removed now, from the revision the pass keeps (A-61, A-67); with none kept, it is
|
||||
// said in the log that it was not said. fp is the want's, and for a removal the fingerprint last wanted there.
|
||||
func (p *passer) sayWanted(node, module, fp, pipeline string, removed bool) {
|
||||
if p.publishWants == nil {
|
||||
return
|
||||
}
|
||||
g := p.graph
|
||||
var kept meshwants.AssignmentRevision
|
||||
var source int64
|
||||
ok := false
|
||||
if g != nil {
|
||||
kept, source, ok = g.keptNow()
|
||||
}
|
||||
if !ok {
|
||||
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: kept.Revision, Source: source, At: &at}
|
||||
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
|
||||
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, kept.Revision, at.Format(time.RFC3339Nano))
|
||||
if err := p.wantsSay(event, body, id); err != nil {
|
||||
p.logf("wanted of %s on %s is not said: %v", module, node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
|
||||
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
|
||||
// version and the parts of the `version-released` whose parts are the digests its build.ended made (A-421): the catalog stamps no
|
||||
// source on an entry (design 47 H §764, H-A-48). A trunk entry whose release is not heard is said, once, naming which was tried.
|
||||
func (p *passer) namedBuild(module, trunkFP string, unreleased ...bool) func(fp string) (assign.KeptBuild, bool) {
|
||||
return func(fp string) (assign.KeptBuild, bool) {
|
||||
if k, ok := p.w.ends.partsOf(module, fp); ok {
|
||||
return k, true
|
||||
}
|
||||
if len(unreleased) > 0 && unreleased[0] && fp == trunkFP {
|
||||
// Wanted with no passed pipeline (the holder of mesh-catalog, the runner's first start): no pipeline released it,
|
||||
// so its want names the parts its build.ended made, until a release of it is heard (design 47 G §16, A-428).
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
if made := p.w.ends.madeOf(module, fp); len(made) > 0 {
|
||||
return assign.KeptBuild{Parts: made}, true
|
||||
}
|
||||
}
|
||||
if fp != trunkFP {
|
||||
// A want held back at an older build (rule 2: no pipeline passed at the trunk's newer fingerprint) names that build's
|
||||
// release when one names exactly the digests it made (novox 2026-11-16: mesh-hand-acts 0.1.13 held at 4eb3cf48 after
|
||||
// the mesh-sdk merge moved its fingerprint).
|
||||
return p.w.released.withParts(module, p.w.ends.madeOf(module, fp))
|
||||
}
|
||||
if _, _, kept := p.graphKept(); !kept {
|
||||
p.unheardRelease(module, fp, "no assignment graph revision is kept yet, so no release is looked up")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
p.unheardRelease(module, fp, "no version-released of it names the parts its build.ended made")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// unheardRelease says once per module, fingerprint and why that a trunk entry's release is not heard, so its wants and candidates
|
||||
// carry no version and no parts (design 47 G A-419).
|
||||
func (p *passer) unheardRelease(module, fp, why string) {
|
||||
key := "release " + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the release of %s at fingerprint %s is not heard: %s; its wants and candidates carry no version and no parts", module, fp, why)
|
||||
}
|
||||
}
|
||||
|
||||
// graphKept is the revision the wants follow, with its trunk source, and whether one is kept; with no graph kept, none is.
|
||||
func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) {
|
||||
if p.graph == nil {
|
||||
return meshwants.AssignmentRevision{}, 0, false
|
||||
}
|
||||
return p.graph.keptNow()
|
||||
}
|
||||
|
||||
// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer
|
||||
// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module).
|
||||
func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
g.failures++
|
||||
n, retry, after := g.failures, g.retry, g.after
|
||||
g.mu.Unlock()
|
||||
if retry <= 0 {
|
||||
retry = defaultKeepingRetry
|
||||
}
|
||||
if after == nil {
|
||||
after = func(d time.Duration, f func()) { time.AfterFunc(d, f) }
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err)
|
||||
if n == keepingFailedAt && p.findings.Put != nil {
|
||||
if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", kindKeepingFailed, err)
|
||||
}
|
||||
}
|
||||
after(retry, func() {
|
||||
g.mu.Lock()
|
||||
newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision)
|
||||
if !newer {
|
||||
g.handedRev, g.handed = c.Revision, true
|
||||
}
|
||||
g.mu.Unlock()
|
||||
if !newer {
|
||||
p.needKeeping(ctx, keepNeed{c: &c})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared.
|
||||
func (p *passer) keepingSucceeded() {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
n := g.failures
|
||||
g.failures = 0
|
||||
g.mu.Unlock()
|
||||
if n >= keepingFailedAt && p.findings.Delete != nil {
|
||||
o := keepingFailedFinding(n, nil)
|
||||
if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil {
|
||||
p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)).
|
||||
func keepingFailedFinding(n int, err error) assign.Observed {
|
||||
summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n)
|
||||
if err != nil {
|
||||
summary += ": " + err.Error()
|
||||
}
|
||||
return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed,
|
||||
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept",
|
||||
Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.",
|
||||
Source: "assigner." + kindKeepingFailed}
|
||||
}
|
||||
@@ -1,395 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// said is one event the passer said on the seat mesh-wants.
|
||||
type said struct {
|
||||
event, id string
|
||||
body json.RawMessage
|
||||
}
|
||||
|
||||
// Design 47 G A-67, §292: the pass keeps an assignment graph revision per catalog revision taken in, before it writes any want, and
|
||||
// says assignment-revision; A-61, §693: a want written or removed is said `wanted` on the subject of its node and module in the same
|
||||
// write, from the kept revision and its catalog's source, a put that wrote nothing says nothing, and with no revision kept none is said.
|
||||
func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
kv := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): kv}, at)
|
||||
p.logf = t.Logf
|
||||
var events []said
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
events = append(events, said{event, id, raw})
|
||||
return nil
|
||||
}
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
// Design 46 approval mark §6: a node's reading keeps the fingerprint of the runner that announced it, as its record says.
|
||||
rec, _ := json.Marshal(nodeengine.HaveByNodeNodeValue{V: 1, Record: &nodeengine.NodeRecord{V: 1, Node: "ace",
|
||||
RunnerCapabilities: []string{"bus-state-calls", "approval-mark/1"}, RunnerFingerprint: "sha256:runner"}})
|
||||
p.w.put(buckets.Have, "ace.node", rec)
|
||||
if c := p.capabilityDigests(); len(c) != 1 || c[0].Node != "ace" || c[0].Fingerprint != "sha256:runner" ||
|
||||
!slices.Equal(c[0].Readings, []string{"approval-mark/1", "bus-state-calls"}) || !strings.HasPrefix(c[0].Digest, "sha256:") {
|
||||
t.Errorf("the capability readings are %+v", c)
|
||||
}
|
||||
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fpTest}}}
|
||||
objects := memObjects{}
|
||||
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} }}
|
||||
want := mwc.Want{Fp: fpTest, Self: true}
|
||||
|
||||
p.putWant(ctx, "ace", "web", want) // no revision kept yet: written, and not said
|
||||
if len(events) != 0 || kv.puts != 1 {
|
||||
t.Fatalf("with no revision kept: %d events, %d puts", len(events), kv.puts)
|
||||
}
|
||||
key, _ := mwc.WantKey("ace", "web")
|
||||
delete(p.w.read(buckets.Wants()), key) // the world forgets the want, as before the first pass
|
||||
kv.puts = 0
|
||||
p.w.put(buckets.Wants(), key, nil)
|
||||
|
||||
p.keepGraph(ctx)
|
||||
if len(events) != 1 || events[0].event != meshwants.EventAssignmentRevision || len(objects) == 0 {
|
||||
t.Fatalf("the pass kept %d objects and said %+v", len(objects), events)
|
||||
}
|
||||
p.keepGraph(ctx) // the same inputs: nothing kept, nothing said
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("a pass of the same inputs said %d events", len(events))
|
||||
}
|
||||
p.putWant(ctx, "ace", "web", want)
|
||||
p.putWant(ctx, "ace", "web", want) // unchanged: not written, not said
|
||||
if len(events) != 2 || kv.puts != 1 {
|
||||
t.Fatalf("a want put twice: %d events, %d puts", len(events), kv.puts)
|
||||
}
|
||||
if wantEvent := "wanted.ace.web"; events[1].event != wantEvent {
|
||||
t.Errorf("wanted said on %q, want %q", events[1].event, wantEvent)
|
||||
}
|
||||
var w meshwants.WantedEvent
|
||||
if err := json.Unmarshal(events[1].body, &w); err != nil || w.Node != "ace" || w.Module != "web" || w.Fp != fpTest || w.Removed || w.Revision != 1 || w.Source != 9 {
|
||||
t.Errorf("wanted = %+v (%v)", w, err)
|
||||
}
|
||||
p.putWant(ctx, "ace", "web", mwc.Want{Removed: true})
|
||||
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
|
||||
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
|
||||
}
|
||||
// Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a
|
||||
// want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands.
|
||||
t.Run("a want's run", func(t *testing.T) {
|
||||
wkey, _ := mwc.WantKey("g14", "db")
|
||||
stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w }
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Pipeline != "main.12" {
|
||||
t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w)
|
||||
}
|
||||
puts := kv.puts
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts {
|
||||
t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts)
|
||||
}
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
|
||||
if kv.puts != puts {
|
||||
t.Error("a want that stands was written again")
|
||||
}
|
||||
skey, _ := mwc.WantKey("shanks", "db")
|
||||
p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"})
|
||||
if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" {
|
||||
t.Errorf("a put's want is %+v, want the newest source held, 9", w)
|
||||
}
|
||||
// a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused
|
||||
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"})
|
||||
if w := stored(); w.Run != "12" || w.Version != "0.1.0" {
|
||||
t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w)
|
||||
}
|
||||
})
|
||||
|
||||
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
|
||||
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
|
||||
// keeps none, and the run's group-ended drops it.
|
||||
before, puts := len(events), kv.puts
|
||||
ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5}
|
||||
var asked []meshcatalog.RevisionInput
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.RevisionInput)
|
||||
asked = append(asked, in)
|
||||
rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}}
|
||||
a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 {
|
||||
t.Errorf("a try's candidate asked %v: %v", asked, err)
|
||||
}
|
||||
if err := p.keepCandidate(ctx, call, ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 {
|
||||
t.Errorf("the candidate's revision was asked as %+v", asked)
|
||||
}
|
||||
a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"})
|
||||
if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 {
|
||||
t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err)
|
||||
}
|
||||
if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" {
|
||||
t.Errorf("the run's entry of web is %+v: %v", e.Entry, err)
|
||||
}
|
||||
if len(events) != before || kv.puts != puts {
|
||||
t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts)
|
||||
}
|
||||
if err := p.dropCandidate(ctx, "run-7"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok {
|
||||
t.Error("the run's group-ended left its candidate kept")
|
||||
}
|
||||
if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil {
|
||||
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
|
||||
// A listing that meets the gate closes entered: the keeping took its need and runs it (H-A-476).
|
||||
type blockedObjects struct {
|
||||
memObjects
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
gate, entered := b.gate, b.entered
|
||||
if gate != nil && entered != nil {
|
||||
close(entered)
|
||||
b.entered = nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
// block stands a gate, opened by the function it answers.
|
||||
func (b *blockedObjects) block() func() {
|
||||
gate := make(chan struct{})
|
||||
b.mu.Lock()
|
||||
b.gate, b.entered = gate, make(chan struct{})
|
||||
b.mu.Unlock()
|
||||
return func() {
|
||||
b.mu.Lock()
|
||||
b.gate, b.entered = nil, nil
|
||||
b.mu.Unlock()
|
||||
close(gate)
|
||||
}
|
||||
}
|
||||
|
||||
// awaitBlocked returns once a keeping meets the standing gate, so what the keeping took is read after it took it, and fails the
|
||||
// test when none did within bound (H-A-476: a test waits on the signal of the work it reads after, never on an order it assumes).
|
||||
func (b *blockedObjects) awaitBlocked(t *testing.T, bound time.Duration, what string) {
|
||||
t.Helper()
|
||||
b.mu.Lock()
|
||||
entered := b.entered
|
||||
b.mu.Unlock()
|
||||
if entered == nil {
|
||||
return // met already
|
||||
}
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(bound):
|
||||
t.Fatalf("no keeping met the gate within %s: %s", bound, what)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
|
||||
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
|
||||
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
|
||||
// more. Each pass and each keeping says its duration.
|
||||
func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
bound := testbus.SetupBound(t) // every wait on a goroutine's work, never a value in the test (H-A-476)
|
||||
var mu sync.Mutex
|
||||
var journal strings.Builder
|
||||
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
|
||||
said := func(words string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return strings.Contains(journal.String(), words)
|
||||
}
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := mcc.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: mcc.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 1, mpr.StepResultOutcomePassed)))
|
||||
objects := &blockedObjects{memObjects: memObjects{}}
|
||||
var kept sync.WaitGroup
|
||||
woke := make(chan struct{}, 8)
|
||||
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
|
||||
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} },
|
||||
background: func(f func()) { kept.Go(f) }, wake: func() { woke <- struct{}{} }}
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
done := make(chan struct{})
|
||||
go func() { p.pass(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the pass waits on the keeping of the assignment graph")
|
||||
}
|
||||
}
|
||||
wantOf := func(node string) (mwc.Want, error) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
return mwc.DecodeWant(wantsKV.values[key])
|
||||
}
|
||||
open := objects.block()
|
||||
t.Cleanup(func() { // the keeping ends before the test does, and says its duration
|
||||
objects.mu.Lock()
|
||||
standing := objects.gate != nil
|
||||
objects.mu.Unlock()
|
||||
if standing {
|
||||
open()
|
||||
}
|
||||
kept.Wait()
|
||||
if !said("the keeping of the assignment graph took ") {
|
||||
t.Error("the keeping does not say its duration")
|
||||
}
|
||||
})
|
||||
|
||||
pass() // the start: the first keeping is blocked
|
||||
if _, err := wantOf("ace"); err == nil {
|
||||
t.Error("a pass at the start wrote a want before an assignment graph revision is kept")
|
||||
}
|
||||
open()
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the first keeping's end wakes no pass")
|
||||
}
|
||||
pass()
|
||||
if w, err := wantOf("ace"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on ace = %+v (%v), want the passed version once a revision is kept", w, err)
|
||||
}
|
||||
|
||||
open = objects.block() // a keeping of new inputs is blocked
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pass()
|
||||
pass() // the same inputs again
|
||||
objects.awaitBlocked(t, bound, "the keeping of new inputs never took its need") // H-A-476: read the need once it was taken
|
||||
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
|
||||
}
|
||||
p.graph.mu.Lock()
|
||||
keeping, waiting := p.graph.keeping, p.graph.need
|
||||
p.graph.mu.Unlock()
|
||||
if !keeping || waiting != nil {
|
||||
t.Errorf("keeping %v, need waiting %+v: the pass of the same inputs handed its revision again", keeping, waiting)
|
||||
}
|
||||
if !said("the pass took ") {
|
||||
t.Error("the pass does not say its duration")
|
||||
}
|
||||
open()
|
||||
kept.Wait()
|
||||
// Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third
|
||||
// failure in a row, cleared at the next that succeeds.
|
||||
t.Run("a failed keeping runs again", func(t *testing.T) {
|
||||
failing := &failingObjects{blockedObjects: objects, fail: 3}
|
||||
var retries []time.Duration
|
||||
var put []mcseat.Finding
|
||||
var deleted []string
|
||||
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil },
|
||||
Delete: func(key string) error { deleted = append(deleted, key); return nil }}
|
||||
p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second
|
||||
p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() }
|
||||
placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed
|
||||
pass()
|
||||
if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" ||
|
||||
len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 {
|
||||
t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures)
|
||||
}
|
||||
if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 {
|
||||
t.Fatal("the revision is not kept once the store takes it")
|
||||
}
|
||||
// keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why.
|
||||
if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil {
|
||||
t.Error("a keeping by hand with no why runs")
|
||||
}
|
||||
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" {
|
||||
t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err)
|
||||
}
|
||||
failing.fail, retries = 2, nil
|
||||
p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) }
|
||||
placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand
|
||||
pass()
|
||||
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 {
|
||||
t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries)
|
||||
}
|
||||
var asked []string
|
||||
tool := keepNowTools(func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
})[0]
|
||||
if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") ||
|
||||
asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` {
|
||||
t.Errorf("assigner_keep_now: %v, asked %v", err, asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// failingObjects refuses the next fail puts, then is objects.
|
||||
type failingObjects struct {
|
||||
*blockedObjects
|
||||
fail int
|
||||
}
|
||||
|
||||
func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
if f.fail > 0 {
|
||||
f.fail--
|
||||
return errors.New("the store answers nothing")
|
||||
}
|
||||
return f.blockedObjects.Put(ctx, name, data)
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the
|
||||
// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name
|
||||
// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each
|
||||
// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings;
|
||||
// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
|
||||
// half, never its own output back (§195).
|
||||
type halves struct {
|
||||
w *world
|
||||
call mcc.Call
|
||||
reader *views.ProcessReader
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
ctx context.Context
|
||||
|
||||
branches *mcc.Branches
|
||||
mu sync.Mutex
|
||||
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
|
||||
settled chan<- struct{}
|
||||
told bool
|
||||
|
||||
// regMu orders the registrations told to registered: the ones held when it is set, then each change (H-A-270 (3)).
|
||||
regMu sync.Mutex
|
||||
registered func(reg meshcatalog.Registration, gone bool)
|
||||
}
|
||||
|
||||
// follow makes the branch follower on the reader; the reader's start asks the first views.
|
||||
func (h *halves) follow() error {
|
||||
b, err := mcc.FollowBranchesProcess(h.reader, h.call)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.Logf = h.logf
|
||||
b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
|
||||
h.changed(branch, module, entry, deleted)
|
||||
}
|
||||
b.Adopted = h.adopted
|
||||
// The one handler of registration-changed is the follower's (H-A-270 (3)): the pipeline ends are told each change from it,
|
||||
// never from a second handler on the hub.
|
||||
reconcile := b.Registrations.Follower.Changed
|
||||
b.Registrations.Follower.Changed = func(module string, value json.RawMessage, gone bool, at time.Time) {
|
||||
if reconcile != nil {
|
||||
reconcile(module, value, gone, at)
|
||||
}
|
||||
h.tellRegistration(module, value, gone)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.branches = b
|
||||
h.mu.Unlock()
|
||||
h.settleWhenNoBranch()
|
||||
return nil
|
||||
}
|
||||
|
||||
// settleWhenNoBranch tells the first pass once, when the registrations and the gate's source are answered and name no branch: the
|
||||
// trunk is none-yet and the world is empty, as the catalog client's Ready says it (design 47 G, A-372). A mesh that names a branch is
|
||||
// told by adopted, when that branch's first view is answered.
|
||||
func (h *halves) settleWhenNoBranch() {
|
||||
if h.settled == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
go func() {
|
||||
select {
|
||||
case <-b.FirstViews(h.ctx):
|
||||
case <-h.ctx.Done():
|
||||
return
|
||||
}
|
||||
if len(b.Followed()) > 0 || b.Ready() != nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
told := h.told
|
||||
h.told = true
|
||||
h.mu.Unlock()
|
||||
if !told {
|
||||
h.settled <- struct{}{}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// first is the first watched branch followed, the trunk's: "" while none is.
|
||||
func (h *halves) first() string {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil {
|
||||
return ""
|
||||
}
|
||||
if f := b.Followed(); len(f) > 0 {
|
||||
return f[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its
|
||||
// first view.
|
||||
func (h *halves) adopted(branch string, c *mcc.Catalog) {
|
||||
first := h.first()
|
||||
if first == "" {
|
||||
return
|
||||
}
|
||||
h.w.setTrunk(first)
|
||||
h.wake()
|
||||
if branch != first || h.settled == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
told := h.told
|
||||
h.told = true
|
||||
h.mu.Unlock()
|
||||
if told {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
select {
|
||||
case <-c.Ready():
|
||||
h.settled <- struct{}{}
|
||||
case <-h.ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// changed takes one entry of branch's half into the world, or a module taken out of it.
|
||||
func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) {
|
||||
defer h.wake()
|
||||
if deleted {
|
||||
h.w.dropModule(branch, module)
|
||||
return
|
||||
}
|
||||
var e meshcatalog.CatalogEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
h.logf("the entry of %s on branch %q does not read: %v", module, branch, err)
|
||||
return
|
||||
}
|
||||
v, n, m := fromEntry(e)
|
||||
h.w.putModule(branch, module, v, n, m)
|
||||
h.w.putEdges(branch, module, e.Edges)
|
||||
}
|
||||
|
||||
// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of
|
||||
// its words.
|
||||
func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) {
|
||||
v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages,
|
||||
Path: e.Path, Repository: e.Repository, Verdict: e.Verdict}
|
||||
n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires,
|
||||
Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
|
||||
for p := range e.Packages {
|
||||
n.Packages = append(n.Packages, p)
|
||||
}
|
||||
slices.Sort(n.Packages)
|
||||
if len(e.Manifest) == 0 {
|
||||
return v, n, nil
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return v, n, nil
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
v.Manifest = "sha256:" + hex.EncodeToString(sum[:])
|
||||
man, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
|
||||
}
|
||||
n.Serves = graph.ServesOf(man)
|
||||
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
|
||||
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
|
||||
return v, n, &parsed{module: e.Module, manifest: man}
|
||||
}
|
||||
|
||||
// views is the catalog graph revision held now of each followed branch whose first view is answered.
|
||||
func (h *halves) views() map[string]mcc.Half {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
out := map[string]mcc.Half{}
|
||||
if b == nil {
|
||||
return out
|
||||
}
|
||||
for _, branch := range b.Followed() {
|
||||
if c, ok := b.Catalog(branch); ok {
|
||||
if half, ok := c.View(); ok {
|
||||
out[branch] = half
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the
|
||||
// order `watch` added it). Once registrations are held, a module none names has none: it is unlisted (H-A-144). Only before the
|
||||
// first registrations are held do the gate's source's branches stand in (the branches followed then), in the order the source
|
||||
// gives them, by their names escaped as its keys are (design 47 G §532, H-A-143, H-A-144).
|
||||
func (h *halves) watchedOf(module string) []string {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if b.Registrations != nil && len(b.Registrations.All()) > 0 {
|
||||
if reg, ok := b.Registrations.Registration(module); ok {
|
||||
return slices.Clone(reg.Branches)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := b.Followed()
|
||||
slices.SortStableFunc(out, func(x, y string) int { return strings.Compare(keys.EscapeName(x), keys.EscapeName(y)) })
|
||||
return out
|
||||
}
|
||||
|
||||
// tellRegistration tells registered of one registration the follower holds now, or of one removed.
|
||||
func (h *halves) tellRegistration(module string, value json.RawMessage, gone bool) {
|
||||
h.regMu.Lock()
|
||||
defer h.regMu.Unlock()
|
||||
if h.registered == nil {
|
||||
return
|
||||
}
|
||||
reg := meshcatalog.Registration{Module: module}
|
||||
if !gone {
|
||||
if err := json.Unmarshal(value, ®); err != nil {
|
||||
h.logf("the registration of %s does not read: %v", module, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
h.registered(reg, gone)
|
||||
}
|
||||
|
||||
// followRegistrations tells registered of every registration the branch follower holds and then of each change it takes
|
||||
// (design 47 G §353): the registrations are followed once, by the follower's one handler of registration-changed (H-A-270
|
||||
// (3)). It says false when no follower is made.
|
||||
func (h *halves) followRegistrations(registered func(meshcatalog.Registration, bool)) bool {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil || b.Registrations == nil {
|
||||
return false
|
||||
}
|
||||
h.regMu.Lock()
|
||||
defer h.regMu.Unlock()
|
||||
h.registered = registered
|
||||
for _, reg := range b.Registrations.All() {
|
||||
registered(reg, false)
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// Design 47 G A-372: the assigner's wait for its catalog half asks the catalog client's Ready as well: a mesh whose registrations and
|
||||
// gate's source are answered and name no branch is none-yet, and the first pass is told so once; before they answer it is not told.
|
||||
func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
if key == mcc.RegistrationsVerb {
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: []seat.Registration{}})
|
||||
}
|
||||
return json.Marshal(map[string][]string{"branches": {}})
|
||||
}
|
||||
for _, answered := range []bool{false, true} {
|
||||
settled := make(chan struct{}, 2)
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx, settled: settled}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if answered {
|
||||
for _, f := range reader.Followers() {
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("a mesh that names no branch never settled the first pass")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
t.Errorf("answered %v: the first pass was told twice or before the first views", answered)
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, H-A-144, G §532: a module's watched branches are its registration's, in its order; once registrations are held a
|
||||
// module none names has none; before the first registrations the gate's source's branches stand in, in the source's order.
|
||||
// H-A-270 (3): the pipeline ends are told the registrations by the same follower, whether set before its first view or after.
|
||||
func TestAModulesWatchedBranchesAreItsRegistrations(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
regs []seat.Registration
|
||||
module string
|
||||
want []string
|
||||
}{
|
||||
{"its registration's order", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "link2pay", []string{"nox-mesh", "main"}},
|
||||
{"no registration names it", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "checker", nil},
|
||||
{"before the first registrations, the source's", []seat.Registration{}, "checker", []string{"main", "master"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case mcc.RegistrationsVerb:
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: tc.regs})
|
||||
case mcc.SourceVerb:
|
||||
return json.Marshal(map[string][]string{"branches": {"master", "main"}})
|
||||
}
|
||||
return nil, errors.New("not answered here")
|
||||
}
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var live, held []string
|
||||
if !h.followRegistrations(func(reg seat.Registration, gone bool) { live = append(live, reg.Module) }) {
|
||||
t.Fatal("no follower of the registrations")
|
||||
}
|
||||
for _, f := range reader.Followers()[:2] { // the registrations and the gate's source
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.followRegistrations(func(reg seat.Registration, gone bool) { held = append(held, reg.Module) })
|
||||
var regs []string
|
||||
for _, r := range tc.regs {
|
||||
regs = append(regs, r.Module)
|
||||
}
|
||||
if !slices.Equal(live, regs) || !slices.Equal(held, regs) {
|
||||
t.Errorf("the ends were told %v at the first view and %v held, want %v", live, held, regs)
|
||||
}
|
||||
if got := h.watchedOf(tc.module); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("the watched branches of %s are %v, want %v", tc.module, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,581 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
|
||||
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
|
||||
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
|
||||
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
|
||||
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
|
||||
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
|
||||
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
|
||||
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
|
||||
|
||||
// inputsTool is the tool that reads the kept inputs.
|
||||
const inputsTool = "assigner_inputs"
|
||||
|
||||
const (
|
||||
// inputsKeptFor is how long an input is kept at the longest.
|
||||
inputsKeptFor = 24 * time.Hour
|
||||
// inputsSwept is how many of the oldest keys each new ask judges.
|
||||
inputsSwept = 20
|
||||
inputsTime = "20060102T150405Z"
|
||||
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
|
||||
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
|
||||
inputsAskPrefix = "ask."
|
||||
)
|
||||
|
||||
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
|
||||
type inputAsk struct {
|
||||
Digest string `json:"digest"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
|
||||
// carries its time in the key itself.
|
||||
func readAsk(raw []byte) (digest string, at time.Time) {
|
||||
var a inputAsk
|
||||
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
|
||||
at, _ = time.Parse(inputsTime, a.At)
|
||||
return a.Digest, at
|
||||
}
|
||||
digest = string(raw)
|
||||
t, _, _ := strings.Cut(digest, ".")
|
||||
at, _ = time.Parse(inputsTime, t)
|
||||
return digest, at
|
||||
}
|
||||
|
||||
// isDigest is whether a key is an input's own key: 64 hex digits.
|
||||
func isDigest(key string) bool {
|
||||
if len(key) != 64 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// digestOf is an input's digest as the ask names it: its sha256 in hex.
|
||||
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
|
||||
|
||||
// inputsCap is the state inputs' cap (design 47 G §354's kept-data row: 64 MiB, the manifest's max-mib): at it, an ask is refused.
|
||||
const inputsCap = 64 << 20
|
||||
|
||||
// inputKeeper keeps the inputs over the state and its object store.
|
||||
type inputKeeper struct {
|
||||
vals stateobjects.Values
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// used is how many bytes the state and its object store hold; nil judges no cap.
|
||||
used func() (int64, error)
|
||||
}
|
||||
|
||||
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
|
||||
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
|
||||
// first, so a sweep never finds the input unnamed.
|
||||
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
|
||||
if ask == "" {
|
||||
return "", errors.New("name the ask the input is kept for")
|
||||
}
|
||||
if err := k.sweep(); err != nil {
|
||||
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
askKey := inputsAskPrefix + keys.EscapeName(ask)
|
||||
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
have, err := k.vals.KV.Get(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if have != nil {
|
||||
return digest, nil
|
||||
}
|
||||
if k.used != nil {
|
||||
used, err := k.used()
|
||||
if err == nil && used+int64(len(body)) > inputsCap {
|
||||
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
|
||||
buckets.InputsState, inputsCap>>20, used, len(body))
|
||||
} else if err != nil {
|
||||
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
|
||||
}
|
||||
if err != nil {
|
||||
_ = k.vals.KV.Delete(askKey)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
|
||||
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
|
||||
func (k *inputKeeper) ended(ask string) error {
|
||||
if ask == "" {
|
||||
return nil
|
||||
}
|
||||
askKey := inputsAskPrefix + keys.EscapeName(ask)
|
||||
e, err := k.vals.KV.Get(askKey)
|
||||
if err != nil || e == nil {
|
||||
return err
|
||||
}
|
||||
digest, _ := readAsk(e.Value)
|
||||
if err := k.vals.KV.Delete(askKey); err != nil {
|
||||
return err
|
||||
}
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if named[digest] {
|
||||
return nil // another running ask still names it
|
||||
}
|
||||
return k.deleteInput(digest)
|
||||
}
|
||||
|
||||
// named is the inputs some ask's key names now.
|
||||
func (k *inputKeeper) named() (map[string]bool, error) {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
named := map[string]bool{}
|
||||
for _, key := range all {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
continue
|
||||
}
|
||||
o, err := k.vals.KV.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o != nil {
|
||||
d, _ := readAsk(o.Value)
|
||||
named[d] = true
|
||||
}
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
|
||||
// ends its object. A key already gone leaves nothing to remove here; the start and assigner_inputs_drop remove what a key left.
|
||||
func (k *inputKeeper) deleteInput(key string) error {
|
||||
e, err := k.vals.KV.Get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
return err
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
if ref, ok := stateobjects.ReadRef(e.Value); ok {
|
||||
if err := k.vals.Objects.Delete(ref.Object); err != nil {
|
||||
return fmt.Errorf("the object %s of %s: %w", ref.Object, key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropKeyGone removes every object of the state whose key is gone and answers their count and bytes (A-465).
|
||||
func (k *inputKeeper) dropKeyGone() (dropped []stateobjects.Kept, bytes int64, err error) {
|
||||
kept, err := k.vals.Kept()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
names, err := k.vals.DropKeyGone()
|
||||
gone := map[string]bool{}
|
||||
for _, n := range names {
|
||||
gone[n] = true
|
||||
}
|
||||
for _, o := range kept {
|
||||
if gone[o.Name] {
|
||||
dropped = append(dropped, o)
|
||||
bytes += o.Size
|
||||
}
|
||||
}
|
||||
return dropped, bytes, err
|
||||
}
|
||||
|
||||
// dropAtStart removes the objects whose key is gone at the program's start, said with their count and bytes (A-465). An object
|
||||
// store not on the bus yet is said and left.
|
||||
func (k *inputKeeper) dropAtStart() {
|
||||
dropped, bytes, err := k.dropKeyGone()
|
||||
if err != nil {
|
||||
k.logf("inputs: the objects of deleted keys are not all removed at the start (%d objects, %d bytes removed): %v", len(dropped), bytes, err)
|
||||
return
|
||||
}
|
||||
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
|
||||
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
|
||||
// input is gone (A-465b).
|
||||
func (k *inputKeeper) sweep() error {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
type askAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var asks []askAt
|
||||
var inputs []string
|
||||
for _, key := range all {
|
||||
if strings.HasPrefix(key, inputsAskPrefix) {
|
||||
e, err := k.vals.KV.Get(key)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
d, at := readAsk(e.Value)
|
||||
if !isDigest(d) {
|
||||
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
|
||||
errs = append(errs, k.vals.KV.Delete(key))
|
||||
}
|
||||
}
|
||||
asks = append(asks, askAt{key, at})
|
||||
continue
|
||||
}
|
||||
if !isDigest(key) {
|
||||
t, _, _ := strings.Cut(key, ".")
|
||||
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
continue
|
||||
}
|
||||
inputs = append(inputs, key)
|
||||
}
|
||||
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
|
||||
if len(asks) > inputsSwept {
|
||||
asks = asks[:inputsSwept]
|
||||
}
|
||||
for _, a := range asks {
|
||||
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
|
||||
errs = append(errs, k.vals.KV.Delete(a.key))
|
||||
}
|
||||
}
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, err)...)
|
||||
}
|
||||
for _, key := range inputs {
|
||||
if !named[key] {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// keyOf is the key of the input of the digest, or none: the digest itself.
|
||||
func (k *inputKeeper) keyOf(digest string) (string, error) {
|
||||
if !isDigest(digest) {
|
||||
return "", nil
|
||||
}
|
||||
e, err := k.vals.KV.Get(digest)
|
||||
if err != nil || e == nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
|
||||
func (k *inputKeeper) body(digest string) ([]byte, error) {
|
||||
if len(digest) != 64 {
|
||||
return nil, errors.New("input takes the input's sha256, 64 hex digits")
|
||||
}
|
||||
key, err := k.keyOf(digest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
|
||||
}
|
||||
var text string
|
||||
found, err := k.vals.Get(key, &text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the input %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(text)
|
||||
if err != nil || digestOf(b) != digest {
|
||||
return nil, fmt.Errorf("the input %s does not hold the bytes its digest names", key)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)
|
||||
|
||||
// verb answers the seat's verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
|
||||
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
|
||||
// size. A caller that does not acknowledge within `within` ends the send, said in logf.
|
||||
func (k *inputKeeper) verb(line buscalls.ChunkLine, within time.Duration) process.SeatHandler {
|
||||
return func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.InputInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
if !callToken.MatchString(in.Call) {
|
||||
return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
|
||||
}
|
||||
digest := strings.TrimPrefix(strings.TrimSpace(in.Digest), "sha256:")
|
||||
body, err := k.body(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: in.Call}
|
||||
total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
||||
// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
|
||||
go func() {
|
||||
if err := send.Send(line, body, within, k.logf); err != nil {
|
||||
k.logf("the send %s of the input %s: %v", in.Call, digest, err)
|
||||
}
|
||||
}()
|
||||
return meshwants.InputAnswer{Digest: digest, Chunks: total}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// inputObjects is the object store beside the state inputs (`inputs-objects`, a state's own, §712), opened when first used: until
|
||||
// the grants part has made it every call is process.ErrNotYet.
|
||||
type inputObjects struct {
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
mu sync.Mutex
|
||||
bus stateobjects.Objects
|
||||
}
|
||||
|
||||
func (o *inputObjects) open() (stateobjects.Objects, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.bus != nil {
|
||||
return o.bus, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object store of the state inputs is not open: %w", err)
|
||||
}
|
||||
o.bus = stateobjects.BusObjects{Store: store}
|
||||
return o.bus, nil
|
||||
}
|
||||
|
||||
func (o *inputObjects) MostBytes() int { return 0 }
|
||||
|
||||
func (o *inputObjects) Put(name string, data []byte) error {
|
||||
b, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Put(name, data)
|
||||
}
|
||||
|
||||
func (o *inputObjects) Get(name string) ([]byte, error) {
|
||||
b, err := o.open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Get(name)
|
||||
}
|
||||
|
||||
func (o *inputObjects) Delete(name string) error {
|
||||
b, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Delete(name)
|
||||
}
|
||||
|
||||
func (o *inputObjects) List() ([]stateobjects.Object, error) {
|
||||
b, err := o.open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.List()
|
||||
}
|
||||
|
||||
// inputsOver is the inputs the program keeps, over its bucket and its object store beside it; nil when the bucket is not opened.
|
||||
func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, logf func(string, ...any)) *inputKeeper {
|
||||
if kv == nil {
|
||||
return nil
|
||||
}
|
||||
objects := &inputObjects{js: js, module: module}
|
||||
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
|
||||
used := func() (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
st, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n := int64(st.Bytes())
|
||||
objs, err := objects.List()
|
||||
if err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
return 0, err
|
||||
}
|
||||
for _, o := range objs {
|
||||
n += o.Size
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
return &inputKeeper{vals: v, now: time.Now, logf: logf, used: used}
|
||||
}
|
||||
|
||||
// inputSaid is one kept input as assigner_inputs lists it.
|
||||
type inputSaid struct {
|
||||
Key string `json:"key"`
|
||||
Time string `json:"time"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
Input string `json:"input,omitempty"`
|
||||
}
|
||||
|
||||
// inputsRead is the tool assigner_inputs over the kept inputs: every kept input with its size, or the one named by digest whole.
|
||||
func inputsRead(k *inputKeeper) stdio.Tool {
|
||||
return stdio.Tool{Name: inputsTool, Description: "The inputs the assigner keeps for the composition steps it asked a build seat to run (design 47 G §354): each " +
|
||||
"with its write time, digest and size, or one by `digest` whole (base64). Read-only.",
|
||||
Input: schema(map[string]string{"digest": "one input's sha256"}, nil), Run: func(args map[string]any) (any, error) {
|
||||
want := arg(args, "digest")
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
|
||||
for _, key := range all {
|
||||
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
|
||||
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
|
||||
lastAsk[d] = at.Format(inputsTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
out := []inputSaid{}
|
||||
for _, key := range all {
|
||||
d := key
|
||||
if !isDigest(key) || (want != "" && d != want) {
|
||||
continue
|
||||
}
|
||||
t := lastAsk[key]
|
||||
var text string
|
||||
if found, err := k.vals.Get(key, &text); err != nil || !found {
|
||||
continue
|
||||
}
|
||||
said := inputSaid{Key: key, Time: t, Digest: d, Size: base64.StdEncoding.DecodedLen(len(text))}
|
||||
if b, err := base64.StdEncoding.DecodeString(text); err == nil {
|
||||
said.Size = len(b)
|
||||
}
|
||||
if want != "" {
|
||||
said.Input = text
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
gone := []map[string]any{}
|
||||
var goneBytes int64
|
||||
answer := map[string]any{"inputs": out}
|
||||
kept, err := k.vals.Kept()
|
||||
if err != nil {
|
||||
// never an empty list for objects that could not be read (A-465)
|
||||
answer["key-gone-error"] = fmt.Sprintf("the objects of the state inputs cannot be listed, so key-gone objects are not known: %v", err)
|
||||
return answer, nil
|
||||
}
|
||||
for _, o := range kept {
|
||||
if o.KeyGone {
|
||||
gone = append(gone, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
|
||||
goneBytes += o.Size
|
||||
}
|
||||
}
|
||||
answer["key-gone"], answer["key-gone-bytes"] = gone, goneBytes
|
||||
return answer, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// inputsDropTool is the tool that removes the objects whose key is gone.
|
||||
const inputsDropTool = "assigner_inputs_drop"
|
||||
|
||||
// inputsDrop is the tool assigner_inputs_drop (A-465): a hand act, recorded first, that removes the objects of the state inputs
|
||||
// whose key is gone now and answers what it removed.
|
||||
func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: inputsDropTool, Description: "Remove the objects of the assigner's state inputs whose key is gone now (design 47 G A-465), " +
|
||||
"and answer which and how many bytes. An object a key names is never removed. Recorded as a hand act; assigner_inputs lists them as key-gone.",
|
||||
Input: schema(map[string]string{"why": "why the objects of deleted keys are dropped now"}, []string{"why"}), Run: func(args map[string]any) (any, error) {
|
||||
why := strings.TrimSpace(arg(args, "why"))
|
||||
if why == "" {
|
||||
return nil, errors.New("say why the objects of deleted keys are dropped")
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: inputsDropTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "inputs-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded, so nothing was removed: %w", err)
|
||||
}
|
||||
dropped, bytes, err := k.dropKeyGone()
|
||||
out := []map[string]any{}
|
||||
for _, o := range dropped {
|
||||
out = append(out, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
|
||||
}
|
||||
return map[string]any{"dropped": out, "count": len(out), "bytes": bytes, "hand-act": taken.Log}, err
|
||||
}}
|
||||
}
|
||||
|
||||
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
|
||||
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
|
||||
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
|
||||
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
|
||||
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var e buildstep.StepEnded
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if runEnded != nil {
|
||||
if err := runEnded(e); err != nil {
|
||||
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := k.ended(e.Ask); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
@@ -1,503 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
)
|
||||
|
||||
type kvMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
// delErr, when set, fails the next Delete once.
|
||||
delErr error
|
||||
}
|
||||
|
||||
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (k *kvMem) Get(key string) (*stateobjects.Entry, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
v, ok := k.vals[key]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return &stateobjects.Entry{Value: v, Revision: k.revs[key]}, nil
|
||||
}
|
||||
func (k *kvMem) Create(key string, v []byte) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; ok {
|
||||
return stateobjects.ErrChanged
|
||||
}
|
||||
k.vals[key], k.revs[key] = v, 1
|
||||
return nil
|
||||
}
|
||||
func (k *kvMem) Update(key string, v []byte, rev uint64) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.revs[key] != rev {
|
||||
return stateobjects.ErrChanged
|
||||
}
|
||||
k.vals[key], k.revs[key] = v, rev+1
|
||||
return nil
|
||||
}
|
||||
func (k *kvMem) Delete(key string) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if err := k.delErr; err != nil {
|
||||
k.delErr = nil
|
||||
return err
|
||||
}
|
||||
delete(k.vals, key)
|
||||
return nil
|
||||
}
|
||||
func (k *kvMem) Keys() ([]string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
var out []string
|
||||
for key := range k.vals {
|
||||
out = append(out, key)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type objectsMem struct {
|
||||
mu sync.Mutex
|
||||
objs map[string][]byte
|
||||
listErr error // List fails with it
|
||||
}
|
||||
|
||||
func (*objectsMem) MostBytes() int { return 0 }
|
||||
func (o *objectsMem) Put(name string, data []byte) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
o.objs[name] = data
|
||||
return nil
|
||||
}
|
||||
func (o *objectsMem) Get(name string) ([]byte, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
d, ok := o.objs[name]
|
||||
if !ok {
|
||||
return nil, errors.New("no such object")
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
func (o *objectsMem) Delete(name string) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
delete(o.objs, name)
|
||||
return nil
|
||||
}
|
||||
func (o *objectsMem) List() ([]stateobjects.Object, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.listErr != nil {
|
||||
return nil, o.listErr
|
||||
}
|
||||
var out []stateobjects.Object
|
||||
for n, d := range o.objs {
|
||||
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
|
||||
type lineMem struct {
|
||||
mu sync.Mutex
|
||||
hear nats.MsgHandler
|
||||
chunks map[string][]byte
|
||||
}
|
||||
|
||||
func (l *lineMem) Publish(subject string, data []byte) error {
|
||||
l.mu.Lock()
|
||||
l.chunks[subject] = data
|
||||
h := l.hear
|
||||
l.mu.Unlock()
|
||||
var c buscalls.SeatChunk
|
||||
if err := json.Unmarshal(data, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
|
||||
l.mu.Lock()
|
||||
l.hear = fn
|
||||
l.mu.Unlock()
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func testKeeper(objs *objectsMem, now func() time.Time) *inputKeeper {
|
||||
return &inputKeeper{vals: stateobjects.Values{KV: newKVMem(), Objects: objs, Store: "inputs-objects"}, now: now, logf: func(string, ...any) {}}
|
||||
}
|
||||
|
||||
// Design 47 G §354, §712, A-237: an input of any size is kept as an object beside the state, and the seat mesh-wants' verb input
|
||||
// answers it in chunks of at most 512 KiB, each acknowledged before the next.
|
||||
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, time.Now)
|
||||
body := make([]byte, 3<<20+17)
|
||||
if _, err := rand.Read(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest, err := k.keep("ask-big", body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs.objs) != 1 {
|
||||
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
|
||||
}
|
||||
if got, err := k.body(digest); err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
|
||||
}
|
||||
line := &lineMem{chunks: map[string][]byte{}}
|
||||
verb := k.verb(line, time.Second)
|
||||
raw, _ := json.Marshal(meshwants.InputInput{Digest: digest, Call: "0123456789abcdef"})
|
||||
out, err := verb("", raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ans := out.(meshwants.InputAnswer)
|
||||
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
||||
if ans.Digest != digest || ans.Chunks != want || want < 6 {
|
||||
t.Fatalf("the answer %+v, want %d chunks", ans, want)
|
||||
}
|
||||
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}
|
||||
var got []byte
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for n := int64(1); n <= want; n++ {
|
||||
for {
|
||||
line.mu.Lock()
|
||||
data, ok := line.chunks[send.Chunk(n)]
|
||||
line.mu.Unlock()
|
||||
if ok {
|
||||
var c buscalls.SeatChunk
|
||||
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
|
||||
t.Fatalf("chunk %d: %+v, %v", n, c, err)
|
||||
}
|
||||
got = append(got, c.Data...)
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("chunk %d of %d never sent", n, want)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(got, body) {
|
||||
t.Error("the chunks do not carry the input's bytes")
|
||||
}
|
||||
if _, err := verb("", json.RawMessage(`{"digest":"`+strings.Repeat("b", 64)+`","call":"0123456789abcdef"}`)); err == nil {
|
||||
t.Error("an input never kept is answered")
|
||||
}
|
||||
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
|
||||
t.Error("a call of no token is answered")
|
||||
}
|
||||
// assigner_inputs lists it with its size and gives one whole by digest
|
||||
list, err := inputsRead(k).Run(map[string]any{})
|
||||
if err != nil || len(list.(map[string]any)["inputs"].([]inputSaid)) != 1 || list.(map[string]any)["inputs"].([]inputSaid)[0].Size != len(body) {
|
||||
t.Errorf("assigner_inputs = %+v %v", list, err)
|
||||
}
|
||||
}
|
||||
|
||||
// §354: an input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, judged at each new ask.
|
||||
func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
|
||||
now := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, func() time.Time { return now })
|
||||
d1, err := k.keep("ask/one", []byte(`{"heads":"abc"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(d1); err != nil {
|
||||
t.Fatalf("a kept input is not read: %v", err)
|
||||
}
|
||||
if err := k.ended("ask/other"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(d1); err != nil {
|
||||
t.Fatalf("the step-ended of another ask deleted it: %v", err)
|
||||
}
|
||||
if err := k.ended("ask/one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(d1); err == nil {
|
||||
t.Error("the input of an ended ask is still answered")
|
||||
}
|
||||
if keys, _ := k.vals.KV.Keys(); len(keys) != 0 {
|
||||
t.Errorf("keys left after the step-ended: %v", keys)
|
||||
}
|
||||
d2, _ := k.keep("ask/two", []byte(`{"heads":"def"}`))
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask/three", []byte(`{"heads":"ghi"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(d2); err == nil {
|
||||
t.Error("an input a day old is still kept at the next ask")
|
||||
}
|
||||
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
|
||||
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
delErr error
|
||||
runErr error
|
||||
naked bool
|
||||
runs int
|
||||
deleted bool
|
||||
}{
|
||||
{name: "the run's end before the input's delete", runs: 1, deleted: true},
|
||||
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
|
||||
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
|
||||
k.vals.KV.(*kvMem).delErr = c.delErr
|
||||
runs := 0
|
||||
runEnded := func(e buildstep.StepEnded) error {
|
||||
runs++
|
||||
if _, err := k.body(d); err != nil {
|
||||
t.Errorf("the run's end is written after the input's delete (%d)", runs)
|
||||
}
|
||||
return c.runErr
|
||||
}
|
||||
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
|
||||
m := &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, runEnded, m, t.Logf)
|
||||
if m.naked != c.naked || m.acked == c.naked {
|
||||
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
|
||||
}
|
||||
if m.naked && c.delErr != nil { // the redelivery
|
||||
m = &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
|
||||
}
|
||||
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
|
||||
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
|
||||
// too; the objects of keys another part deleted are removed by dropKeyGone and by the tool assigner_inputs_drop, which lists
|
||||
// them as key-gone first, and never removes an object a key names.
|
||||
func TestAnEndedInputEndsItsObjectAndTheDropToolRemovesWhatAKeyLeft(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
big := func(b byte) []byte { return bytes.Repeat([]byte{b}, 2<<20) }
|
||||
if _, err := k.keep("ask-one", big('a')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.keep("ask-two", big('b')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs.objs) != 2 {
|
||||
t.Fatalf("objects = %d, want 2", len(objs.objs))
|
||||
}
|
||||
if err := k.ended("ask-one"); err != nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("the step-ended left %d objects (%v), want 1", len(objs.objs), err)
|
||||
}
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask-three", big('c')); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs.objs) != 1 {
|
||||
t.Fatalf("after the sweep of the day-old input %d objects, want only the new one", len(objs.objs))
|
||||
}
|
||||
// An object whose key another part deleted is listed as key-gone and removed by the tool; a live one stays.
|
||||
keys, _ := k.vals.KV.Keys()
|
||||
for _, key := range keys {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
live := []byte(`{"heads":"x"}`)
|
||||
if _, err := k.keep("ask-four", append(live, make([]byte, 2<<20)...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := inputsRead(k).Run(map[string]any{})
|
||||
if err != nil || len(list.(map[string]any)["key-gone"].([]map[string]any)) != 1 {
|
||||
t.Fatalf("assigner_inputs key-gone = %+v %v, want the one object of the deleted key", list, err)
|
||||
}
|
||||
ask := func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"answer":{"log":"hand-acts"}}`), nil
|
||||
}
|
||||
if _, err := inputsDrop(k, ask).Run(map[string]any{}); err == nil {
|
||||
t.Error("dropped with no reason")
|
||||
}
|
||||
out, err := inputsDrop(k, ask).Run(map[string]any{"why": "a key left its object"})
|
||||
if err != nil || out.(map[string]any)["count"] != 1 || len(objs.objs) != 1 {
|
||||
t.Fatalf("assigner_inputs_drop = %+v %v, %d objects left: want one removed and the live one kept", out, err, len(objs.objs))
|
||||
}
|
||||
// The start drops what a key left, said with its count and bytes.
|
||||
if _, err := k.keep("ask-five", live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range keysOf(t, k) {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
_ = k.vals.KV.Delete(key)
|
||||
}
|
||||
}
|
||||
var logged []string
|
||||
k.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
before := len(objs.objs)
|
||||
k.dropAtStart()
|
||||
if len(objs.objs) != 0 || before == 0 || len(logged) != 1 || !strings.Contains(logged[0], fmt.Sprintf("%d objects of deleted keys removed at the start", before)) || !strings.Contains(logged[0], " bytes") {
|
||||
t.Errorf("the start left %d of %d objects and said %q: want every key-gone object removed and its count and bytes said", len(objs.objs), before, logged)
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(t *testing.T, k *inputKeeper) []string {
|
||||
t.Helper()
|
||||
keys, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Design 47 G A-465: assigner_inputs says why key-gone objects are not known when the objects cannot be listed, and never
|
||||
// answers them as none.
|
||||
func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
|
||||
objs := &objectsMem{objs: map[string][]byte{}, listErr: errors.New("the object store does not answer")}
|
||||
k := testKeeper(objs, time.Now)
|
||||
out, err := inputsRead(k).Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if msg, _ := m["key-gone-error"].(string); !strings.Contains(msg, "does not answer") {
|
||||
t.Errorf("answer %+v: want key-gone-error naming the failure", m)
|
||||
}
|
||||
if _, has := m["key-gone"]; has {
|
||||
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
|
||||
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
|
||||
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
|
||||
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
body := bytes.Repeat([]byte{'n'}, 2<<20)
|
||||
var digests []string
|
||||
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
|
||||
d, err := k.keep(ask, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests = append(digests, d)
|
||||
}
|
||||
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
|
||||
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
|
||||
}
|
||||
if _, err := k.vals.KV.Get(digests[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
|
||||
}
|
||||
for _, key := range keysOf(t, k) {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
|
||||
t.Errorf("a key %q beside the one input", key)
|
||||
}
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
|
||||
t.Errorf("the ask does not name the digest it uses: %v", e)
|
||||
}
|
||||
for _, ask := range []string{"ask-1", "ask-2"} {
|
||||
if err := k.ended(ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
|
||||
}
|
||||
}
|
||||
if err := k.ended("ask-3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
|
||||
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
|
||||
}
|
||||
// a day after its last write, a renewed write counted
|
||||
if _, err := k.keep("ask-4", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-5", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-6", []byte("other")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Errorf("an input written again 20 hours ago was swept: %v", err)
|
||||
}
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask-7", []byte("another")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
|
||||
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
|
||||
}
|
||||
// an input of the old form goes a day after the time its key carries, with its object, never earlier
|
||||
old := "20261117T090000Z." + digestOf([]byte("old"))
|
||||
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
|
||||
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.sweep(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
|
||||
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(recent); e == nil {
|
||||
t.Error("an input of the old form under a day old was deleted early")
|
||||
}
|
||||
// a new content the cap refuses says the fill; the same content, already kept, needs no room
|
||||
full := newTestKeeperWithFill(objs, now, inputsCap-10)
|
||||
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
|
||||
t.Fatalf("a content that fits is refused: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
|
||||
t.Errorf("the refusal does not name the fill: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
|
||||
t.Errorf("the content already kept is refused at the cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
k.used = func() (int64, error) { return used, nil }
|
||||
return k
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
@@ -16,31 +11,23 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
|
||||
if viewsEmitter(nil, process.Account{}, nil, &heldWriter{}, t.Logf).Findings == nil {
|
||||
t.Error("the assigner's views emitter writes no view-unsaid finding")
|
||||
}
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
@@ -49,20 +36,20 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reads []string
|
||||
for _, r := range m.Reads {
|
||||
if slices.Contains(readFamilies, r) {
|
||||
continue // a state kept one bucket per node, read by its prefix (ADR 0593 §6), not one bucket
|
||||
if slices.Contains(readLogs, r) {
|
||||
continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket
|
||||
}
|
||||
reads = append(reads, strings.Replace(r, ".", "_", 1))
|
||||
}
|
||||
for _, l := range readFamilies {
|
||||
for _, l := range readLogs {
|
||||
if !slices.Contains(m.Reads, l) {
|
||||
t.Errorf("the program reads the per-node state %s, the manifest does not name it", l)
|
||||
t.Errorf("the program reads the log %s, the manifest does not name it", l)
|
||||
}
|
||||
}
|
||||
var states []string
|
||||
@@ -72,7 +59,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, pair := range []struct {
|
||||
what string
|
||||
code, want []string
|
||||
}{{"reads", readBuckets, reads}, {"states", append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()), states}} { // the marks are bound by the views family (views.go)
|
||||
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
|
||||
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
|
||||
@@ -82,28 +69,35 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, s := range keptStates {
|
||||
kept = append(kept, m.Module+"_"+s.Name)
|
||||
}
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(append(slices.Clone(ownBuckets()), marksBucket(), firstViewsBucket()))); !reflect.DeepEqual(got, want) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
|
||||
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
|
||||
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
lease []byte
|
||||
said string
|
||||
}{
|
||||
{buckets.Puts(), jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{buckets.Puts(), errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{buckets.Puts(), context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{buckets.Wants(), errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
|
||||
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
|
||||
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets())-1 || !strings.Contains(logged, c.said)):
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
@@ -114,21 +108,19 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
wrote := make(chan string, 8)
|
||||
kvs := kvSet{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets()...) {
|
||||
kvs := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
placed, err := mmac.AssignmentsValues(mma.Assignments{Node: "ace"})
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[plc.AssignmentsBucket()].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[buckets.Puts()].(*watchKV).closed = true
|
||||
kvs[buckets.Wants()].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, nil, seatlist.NewPart("assigner"), log.New(io.Discard, "", 0), func(string) string { return "" })
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
@@ -144,13 +136,18 @@ type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
lease []byte // the controller's lease, when it holds one
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
kv := &memKV{values: map[string][]byte{}}
|
||||
if name == buscompose.LeaseBucket && r.lease != nil {
|
||||
kv.values[buscompose.ControllerLeaseKey] = r.lease
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
@@ -198,63 +195,3 @@ func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
func (e kvEntry) Revision() uint64 { return 1 }
|
||||
func (e kvEntry) Created() time.Time { return time.Time{} }
|
||||
|
||||
// Keys lists nothing: the retired state this watch stands for holds no key.
|
||||
func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
|
||||
return nil, jetstream.ErrNoKeysFound
|
||||
}
|
||||
|
||||
// putIntoMu holds putInto's writes one at a time: the composition step starts its machines' runs side by side, and each may
|
||||
// raise a finding into the same map.
|
||||
var putIntoMu sync.Mutex
|
||||
|
||||
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
|
||||
func putInto(into map[string][]byte) func(mcseat.Finding) error {
|
||||
return func(f mcseat.Finding) error {
|
||||
raw, err := json.Marshal(f)
|
||||
putIntoMu.Lock()
|
||||
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
|
||||
putIntoMu.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// fakeWire is a views part's finding writer that hands each written finding's kind on.
|
||||
type fakeWire struct{ written chan string }
|
||||
|
||||
func (f fakeWire) WriteWire(_ context.Context, finding map[string]any) error {
|
||||
k, _ := finding["kind"].(string)
|
||||
f.written <- k
|
||||
return nil
|
||||
}
|
||||
func (fakeWire) Clear(context.Context, string, string, string, string) error { return nil }
|
||||
func (fakeWire) ModuleNode() (string, string) { return "mesh-delivery-assigner", "n1" }
|
||||
|
||||
// H-A-60: the assigner's entries are asked again to the setting entry-asks, then written through its finding writer as the
|
||||
// finding entry-unanswered.
|
||||
func TestAnUnreachedEntryEndsInTheAssignersFindingEntryUnanswered(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
unreached := func(string, any) (json.RawMessage, error) { return nil, errors.New("no responders") }
|
||||
w := fakeWire{written: make(chan string, 4)}
|
||||
e := entriesOf(nil, unreached, 1, w, func() {})
|
||||
if e.Asks != 1 {
|
||||
t.Fatalf("entry-asks %d, want 1 from the setting", e.Asks)
|
||||
}
|
||||
e.Need(ctx, "src2:u")
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case kind := <-w.written:
|
||||
if kind != meshcatalogclient.EntryUnansweredKind {
|
||||
t.Fatalf("the finding written is %q, want %s", kind, meshcatalogclient.EntryUnansweredKind)
|
||||
}
|
||||
return
|
||||
default:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no finding entry-unanswered is written past the bound")
|
||||
}
|
||||
e.AskAgain(ctx)
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
)
|
||||
|
||||
// The state this seat's assignments were kept in before ADR 0641 gave it its name. The assigner reads it here and nowhere
|
||||
// else: at its start it copies each key into `assignments` (copyRetired) and keeps no reader of it afterwards. The
|
||||
// manifest still declares the state, so the assigner's account may read it for the copy; a later release drops it.
|
||||
const retiredState = "placement"
|
||||
|
||||
func retiredBucket() string { return plc.Module() + "_" + retiredState }
|
||||
|
||||
// CopiedEvent is the module's event saying the copy was made, once: mesh.mod.mesh-delivery-assigner.event.assignments-copied.
|
||||
const CopiedEvent = "assignments-copied"
|
||||
|
||||
// copiedSaid is what the event carries: the keys copied, those already in `assignments` (kept as they are), and the keys of
|
||||
// the form `placement.<node>` that were dropped (their node key holds the value).
|
||||
type copiedSaid struct {
|
||||
Copied int `json:"copied"`
|
||||
Present int `json:"present"`
|
||||
Dropped int `json:"dropped"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
// copiesState is the assigner's own state of its copies, read by its tool assigner_read_copies, and copiesBucket its bucket.
|
||||
// The retired state gets nothing written: readers of other modules still parse every key in it.
|
||||
const copiesState = "copies"
|
||||
|
||||
func copiesBucket() string { return plc.Module() + "_" + copiesState }
|
||||
|
||||
// copiedMarker is the key, in the state copies, that says the copy ended: its time and the keys it copied. It is written
|
||||
// after the last key, and the copy is made at every start until it is there and never after, so a copy cut short by a
|
||||
// crash is completed at the next start and a key `assignments` dropped after the copy ended is never brought back.
|
||||
const copiedMarker = "copied-from-retired"
|
||||
|
||||
type copiedMark struct {
|
||||
At time.Time `json:"at"`
|
||||
Copied int `json:"copied"`
|
||||
}
|
||||
|
||||
// copyRetired is the holder's switch from the retired state to `assignments`, with no gap: until the marker is there, it
|
||||
// creates in `assignments` each key of the retired state in the form `<node>.node` or `<node>.<module>` that is missing
|
||||
// there (never overwriting one), the module keys first and the node keys, which commit a node, last, and then writes the
|
||||
// marker. It deletes nothing: the readers delivered before this one still read the retired state, and a delete of a
|
||||
// `<node>.node` key there is a node leaving (ADR 0641 section 4); the state is dropped in the next release. Keys of the
|
||||
// form `placement.<node>` are not copied, and a key that fits no form is left and counted. It answers whether the seat may
|
||||
// go on: a copy that failed is tried again at the next pass.
|
||||
func (p *passer) copyRetired(ctx context.Context) bool {
|
||||
if p.copied {
|
||||
return true
|
||||
}
|
||||
old, next, copies := p.kvs[retiredBucket()], p.kvs[plc.AssignmentsBucket()], p.kvs[copiesBucket()]
|
||||
if next == nil || copies == nil {
|
||||
return false
|
||||
}
|
||||
if old == nil {
|
||||
p.logf("assignments: the retired state %s cannot be read, so nothing is copied from it", retiredBucket())
|
||||
p.copied = true
|
||||
return true
|
||||
}
|
||||
switch _, err := copies.Get(ctx, copiedMarker); {
|
||||
case err == nil:
|
||||
p.copied = true // the copy ended at an earlier start: nothing is copied again
|
||||
return true
|
||||
case !errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
p.logf("assignments: the marker %s of %s cannot be read, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
|
||||
return false
|
||||
}
|
||||
names, err := old.Keys(ctx)
|
||||
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
p.logf("assignments: the keys of %s cannot be listed, and are tried again at the next pass: %v", retiredBucket(), err)
|
||||
return false
|
||||
}
|
||||
// A node's module keys before its node key: the node key is the commit the readers wait for.
|
||||
slices.SortStableFunc(names, func(a, b string) int {
|
||||
return strings.Compare(sortForm(a), sortForm(b))
|
||||
})
|
||||
said := copiedSaid{}
|
||||
for _, key := range names {
|
||||
_, _, kind, perr := mmac.ParseKey(key)
|
||||
switch {
|
||||
case strings.HasPrefix(key, retiredState+"."):
|
||||
// The node named placement could not exist before, so this is the form that held a whole node in one key.
|
||||
said.Dropped++
|
||||
case perr != nil || (kind != mmac.NodeKind && kind != mmac.ModuleKind):
|
||||
said.Skipped++
|
||||
continue
|
||||
default:
|
||||
entry, err := old.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("assignments: %s cannot be read, and the copy is tried again at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if entry.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
rev, err := next.Create(ctx, key, entry.Value())
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyExists):
|
||||
said.Present++
|
||||
case err != nil:
|
||||
p.logf("assignments: %s is not copied, and the copy is tried again at the next pass: %v", key, err)
|
||||
return false
|
||||
default:
|
||||
said.Copied++
|
||||
p.w.putEntry(plc.AssignmentsBucket(), key, entry.Value(), rev, p.clock())
|
||||
}
|
||||
}
|
||||
}
|
||||
mark, _ := json.Marshal(copiedMark{At: p.clock(), Copied: said.Copied + said.Present})
|
||||
if _, err := copies.Put(ctx, copiedMarker, mark); err != nil {
|
||||
p.logf("assignments: the end of the copy is not written as %s in %s, and the copy is tried again at the next pass: %v", copiedMarker, copiesBucket(), err)
|
||||
return false
|
||||
}
|
||||
p.copied = true
|
||||
if said == (copiedSaid{}) {
|
||||
return true // the retired state holds nothing
|
||||
}
|
||||
p.logf("assignments: copied %d keys of %s into %s (%d already there kept as they are, %d of the form placement.<node> not copied, %d fitting no form left)",
|
||||
said.Copied, retiredBucket(), plc.AssignmentsBucket(), said.Present, said.Dropped, said.Skipped)
|
||||
if p.seat != nil {
|
||||
if err := p.seat.bus.Emit(CopiedEvent, said); err != nil {
|
||||
p.logf("assignments: the copy is not said as an event: %v", err)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sortForm orders `<node>.<module>` keys before `<node>.node`.
|
||||
func sortForm(key string) string {
|
||||
if node, ok := strings.CutSuffix(key, ".node"); ok {
|
||||
return node + "\x7f"
|
||||
}
|
||||
return key
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// The module claiming the seat node-engine is found by its served verbs, in the catalog's form `seat:<seat>.<verb>`
|
||||
// and in the bare form (live, 2026-11-16: every composition step said "holds no module claiming the seat node-engine").
|
||||
func TestTheNodeEngineEntryIsFoundByItsServedVerbs(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
serves []string
|
||||
found bool
|
||||
}{
|
||||
{"the catalog's seat form", []string{"seat:node-engine.compose", "seat:node-engine.acts"}, true},
|
||||
{"the bare form", []string{"node-engine.compose"}, true},
|
||||
{"another seat", []string{"seat:node-engine-x.compose", "seat:mesh-wants.ask"}, false},
|
||||
} {
|
||||
e, ok := nodeEngineEntry(map[string]meshcatalog.CatalogEntry{"other": {Serves: []string{"seat:mesh-builds.ask"}}, "engine": {Serves: c.serves}})
|
||||
if ok != c.found || (ok && e.Serves[0] != c.serves[0]) {
|
||||
t.Errorf("%s: found %v (%v), want %v", c.name, ok, e.Serves, c.found)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// busObjects is the assigner's own object bucket assignment-revisions in the old `objects` section (novox/hq ADR 0619 §2),
|
||||
// which the grants part creates, as the passer's process reaches it: opened at its first use and kept; until the bucket exists
|
||||
// every call is an error saying so, and the next call opens it again (design 47 G A-67, modelled on the indexer's busObjects).
|
||||
type busObjects struct {
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
local string
|
||||
mu sync.Mutex
|
||||
o jetstream.ObjectStore
|
||||
}
|
||||
|
||||
// newBusObjects is the object bucket of this local name of the module the account runs as.
|
||||
func newBusObjects(js jetstream.JetStream, cred process.Account, local string) *busObjects {
|
||||
return &busObjects{js: js, module: cred.Module, local: local}
|
||||
}
|
||||
|
||||
func (b *busObjects) open(ctx context.Context) (jetstream.ObjectStore, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.o != nil {
|
||||
return b.o, nil
|
||||
}
|
||||
o, err := b.js.ObjectStore(ctx, catalogue.ObjectsBucket(b.module, b.local))
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, fmt.Errorf("the object bucket %s is not on the bus yet (the grants part creates it from the manifest's objects): %w", b.local, process.ErrNotYet)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object bucket %s is not open: %w", b.local, err)
|
||||
}
|
||||
b.o = o
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (b *busObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name}, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *busObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := o.GetBytes(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return nil, assigngraph.ErrNoObject
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (b *busObjects) Names(ctx context.Context) ([]string, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos, err := o.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return []string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make([]string, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
names = append(names, i.Name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// PutDescribed stores data under name with the entry's module and fingerprint in its metadata, which the listing says (assigngraph.Describer,
|
||||
// design 47 H-A-259).
|
||||
func (b *busObjects) PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name, Metadata: map[string]string{entryModuleMeta: module, entryFpMeta: fp}}, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
// Described is every object with the module and fingerprint its metadata says, in one listing.
|
||||
func (b *busObjects) Described(ctx context.Context) ([]assigngraph.Described, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos, err := o.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return []assigngraph.Described{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]assigngraph.Described, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
out = append(out, assigngraph.Described{Name: i.Name, Module: i.Metadata[entryModuleMeta], Fp: i.Metadata[entryFpMeta]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The metadata keys of an entry object's module and fingerprint.
|
||||
const (
|
||||
entryModuleMeta = "module"
|
||||
entryFpMeta = "fp"
|
||||
)
|
||||
|
||||
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
|
||||
func (b *busObjects) Delete(ctx context.Context, name string) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
|
||||
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
st, err := o.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
|
||||
if err != nil {
|
||||
return int64(st.Size()), 0, err
|
||||
}
|
||||
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
|
||||
}
|
||||
@@ -1,460 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/outside"
|
||||
)
|
||||
|
||||
// The wait on an outside cause (design 47 G §706, ADR 0677): the assigner judges a node's health after a want moves, and a moved
|
||||
// module's failing check on a machine whose open conditions name an outside cause is held as a wait (internal/outside has the
|
||||
// rules). The conditions come only through the seat mesh-conditions: its verb `conditions`, read at each judging as a kept
|
||||
// call, and its events, which only say when to judge again.
|
||||
|
||||
// conditionsVerb is the seat verb the keeper answers `conditions` by.
|
||||
const conditionsVerb = "seat:" + mcseat.Seat + "." + mcseat.VerbConditions
|
||||
|
||||
// wantsPublisher says one of the seat mesh-wants' events, `wait-held` and `wait-ended` among them, under a message id the bus
|
||||
// keeps one of.
|
||||
func wantsPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
|
||||
return func(event string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatdef.EventSubject(meshwants.Seat, event))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", assignment.Module())
|
||||
msg.Header.Set("x-node", node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// outsideActs keeps, while the process runs, what the judging of outside waits needs beside the kept waits: the conditions read
|
||||
// for each machine (one answer is one judging), the machines a heard event says to judge again, the machines whose own network
|
||||
// is held open, and each check's last signature.
|
||||
type outsideActs struct {
|
||||
mu sync.Mutex
|
||||
call func(key string, body any) (json.RawMessage, error) // a kept call
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// bound and settle are the settings outside-wait-minutes and outside-settle-seconds.
|
||||
bound, settle time.Duration
|
||||
// reading is a read in flight by machine, answered the answers it brought, not yet judged by.
|
||||
reading map[string]bool
|
||||
answered map[string][]outside.Cond
|
||||
// dirty is a machine whose waits are judged again, cleared a wait's key whose cause's clearing was heard.
|
||||
dirty map[string]bool
|
||||
cleared map[string]bool
|
||||
// sigs is each check's last signature, by node.module.
|
||||
sigs map[string]string
|
||||
// causes is, by condition key, the nodes whose waits stand on it, as the last pass kept them.
|
||||
causes map[string][]string
|
||||
// started says the first pass judged every wait again.
|
||||
started bool
|
||||
}
|
||||
|
||||
func newOutsideActs(call func(string, any) (json.RawMessage, error), wake func(), logf func(string, ...any), bound, settle time.Duration) *outsideActs {
|
||||
if bound <= 0 {
|
||||
bound = outside.DefaultBound
|
||||
}
|
||||
if settle <= 0 {
|
||||
settle = outside.DefaultSettle
|
||||
}
|
||||
return &outsideActs{call: call, wake: wake, logf: logf, bound: bound, settle: settle, reading: map[string]bool{}, answered: map[string][]outside.Cond{},
|
||||
dirty: map[string]bool{}, cleared: map[string]bool{}, sigs: map[string]string{}, causes: map[string][]string{}}
|
||||
}
|
||||
|
||||
// outsideSettings are the bounds of the wait, the settings outside-wait-minutes and outside-settle-seconds (design 47 G §633, §706),
|
||||
// each read at the start from the file the manifest declares with the content ${config:<key>}, which the environment names
|
||||
// (A-112, as the builder reads its own); zero for one whose file is missing or empty, which is its default, and an error
|
||||
// naming the file for a content that is no positive whole number.
|
||||
func outsideSettings(getenv func(string) string) (bound, settle time.Duration, err error) {
|
||||
minutes, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_WAIT_FILE"))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
seconds, err := settingFile(getenv("MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return time.Duration(minutes) * time.Minute, time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
|
||||
// settingFile reads one setting's file: 0 for no path, no file or an empty one.
|
||||
func settingFile(path string) (int, error) {
|
||||
if path == "" {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Heard is one of the seat's events condition-raised, -changed or -cleared: it says which machines to judge again; nothing is kept
|
||||
// of the condition (the network hold ends, design 47 H §750, H-A-55).
|
||||
func (o *outsideActs) Heard(subject string, body []byte) {
|
||||
var e mcseat.ConditionEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Key == "" {
|
||||
o.logf("%s holds what is not a condition's event, and is dropped: %v", subject, err)
|
||||
return
|
||||
}
|
||||
cleared := strings.HasSuffix(subject, "."+mcseat.EventConditionCleared)
|
||||
o.mu.Lock()
|
||||
if cleared {
|
||||
for _, node := range o.causes[e.Key] {
|
||||
o.dirty[node] = true
|
||||
o.cleared[node+"|"+e.Key] = true
|
||||
}
|
||||
} else if e.Subject != nil {
|
||||
o.dirty[e.Subject.Machine] = true
|
||||
for _, m := range e.Subject.Also {
|
||||
o.dirty[m] = true
|
||||
}
|
||||
}
|
||||
o.mu.Unlock()
|
||||
o.wake()
|
||||
}
|
||||
|
||||
// take is the conditions read for machine, consumed (one answer is one judging), or false while the kept call is unanswered,
|
||||
// which it starts when none is in flight.
|
||||
func (o *outsideActs) take(machine string) ([]outside.Cond, bool) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if c, ok := o.answered[machine]; ok {
|
||||
delete(o.answered, machine)
|
||||
return c, true
|
||||
}
|
||||
if o.reading[machine] {
|
||||
return nil, false
|
||||
}
|
||||
o.reading[machine] = true
|
||||
go func() {
|
||||
answer, err := o.call(conditionsVerb, map[string]any{"scope": "machine", "machine": machine})
|
||||
var conds []outside.Cond
|
||||
if err == nil {
|
||||
conds, err = outside.ParseConditions(answer)
|
||||
}
|
||||
o.mu.Lock()
|
||||
o.reading[machine] = false
|
||||
if err != nil {
|
||||
o.logf("the conditions of %s are not read (%v): the move waits on %s", machine, err, outside.OnUnread)
|
||||
} else {
|
||||
o.answered[machine] = conds
|
||||
o.dirty[machine] = true // the kept call answered: the move is judged by it
|
||||
}
|
||||
o.mu.Unlock()
|
||||
if err == nil {
|
||||
o.wake()
|
||||
}
|
||||
}()
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// outsideReview is the judging of module's check at fp on each of its nodes (§706): it returns the nodes as the rules read them,
|
||||
// the have of a waiting move no failure (its health unknown) and a go-on's counted healthy, and keeps the waits, says their
|
||||
// events, finding and ask. moved says the delivery moved owner to node.
|
||||
func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []string, n assign.Nodes, keeperSelf bool, moved func(owner, node string) bool) assign.Nodes {
|
||||
o := p.outside
|
||||
if o == nil || fp == "" || p.kvs[buckets.Waits()] == nil {
|
||||
return n
|
||||
}
|
||||
hex, err := catalogkey.KeyPart(fp)
|
||||
if err != nil {
|
||||
return n
|
||||
}
|
||||
now := p.clock()
|
||||
kept := p.w.read(buckets.Waits())
|
||||
override := map[string]string{}
|
||||
// A wait of another fingerprint of this module is replaced by the newer want.
|
||||
for key, raw := range kept {
|
||||
if w, err := outside.Decode(raw); err == nil && w.Module == module && w.FP != fp {
|
||||
p.outsideEnd(ctx, key, w, outside.HowReplaced, now)
|
||||
}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
key := outside.Key(node, module, hex)
|
||||
var prev *outside.Wait
|
||||
if raw := kept[key]; raw != nil {
|
||||
if w, err := outside.Decode(raw); err == nil {
|
||||
prev = &w
|
||||
}
|
||||
}
|
||||
h, ok := n.Haves[node][module]
|
||||
sig := fmt.Sprintf("%s|%s|%s", h.Fp, h.Phase, h.Health)
|
||||
o.mu.Lock()
|
||||
first := !o.started
|
||||
changed := o.sigs[node+"."+module] != sig
|
||||
o.sigs[node+"."+module] = sig
|
||||
dirty := o.dirty[node]
|
||||
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
|
||||
o.mu.Unlock()
|
||||
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
|
||||
if !failing && !healthy && prev == nil {
|
||||
continue
|
||||
}
|
||||
if prev != nil && p.outsideAnswered(prev, node, module, fp) {
|
||||
dirty = true
|
||||
}
|
||||
due := prev != nil && (prev.Release != "" || (prev.Settling && !now.Before(prev.SettleEnds)))
|
||||
var d outside.Decision
|
||||
if prev != nil && !(first || changed || dirty || due) {
|
||||
// nothing to judge by: the wait stands as kept; the bound's ask is made when it has run
|
||||
d = outside.Decision{Wait: prev, Waiting: prev.Standing() || prev.Settling, Own: prev.Decided == outside.HowFailed || prev.Decided == outside.HowOwnFailure,
|
||||
Counted: prev.Decided == outside.HowGoOn}
|
||||
if prev.Standing() && !now.Before(prev.ClockFrom.Add(o.bound)) {
|
||||
d.Ask = true
|
||||
}
|
||||
} else {
|
||||
f := outside.Facts{Now: now, Failing: failing, Healthy: healthy, Node: node, KeeperSelf: keeperSelf, Bound: o.bound, Settle: o.settle,
|
||||
Moved: func(owner string) bool { return moved(owner, node) }, CauseCleared: cleared}
|
||||
if failing && !keeperSelf {
|
||||
f.Conds, f.Read = o.take(node)
|
||||
}
|
||||
d = outside.Judge(prev, node, module, fp, f)
|
||||
o.mu.Lock()
|
||||
delete(o.dirty, node)
|
||||
if prev != nil {
|
||||
delete(o.cleared, node+"|"+prev.Cause)
|
||||
}
|
||||
o.mu.Unlock()
|
||||
}
|
||||
switch {
|
||||
case d.Waiting:
|
||||
override[node] = nodeengineclient.HealthUnknown
|
||||
case d.Counted:
|
||||
override[node] = nodeengineclient.HealthOK
|
||||
}
|
||||
p.outsideApply(ctx, key, hex, node, module, fp, prev, d, now)
|
||||
}
|
||||
o.mu.Lock()
|
||||
o.started = true
|
||||
o.mu.Unlock()
|
||||
if len(override) == 0 {
|
||||
return n
|
||||
}
|
||||
out := n
|
||||
out.Haves = map[string]map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for node, byModule := range n.Haves {
|
||||
out.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for m, h := range byModule {
|
||||
out.Haves[node][m] = h
|
||||
}
|
||||
}
|
||||
for node, health := range override {
|
||||
h := out.Haves[node][module]
|
||||
h.Health = health
|
||||
if health == nodeengineclient.HealthOK {
|
||||
h.Phase = nodeengineclient.PhaseRunning
|
||||
}
|
||||
out.Haves[node][module] = h
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// outsideAnswered reads the operator's answer to the wait's ask, from the teller's record of it under the ask's key (answeredWith,
|
||||
// §700's path as the assigner reads an ask's answer), and notes it on the wait as a release; it says whether it did.
|
||||
func (p *passer) outsideAnswered(w *outside.Wait, node, module, fp string) bool {
|
||||
if w.Ask == "" || w.Release != "" {
|
||||
return false
|
||||
}
|
||||
key, err := asks.OutsideAskKey(node, module, fp)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, how := range []string{assign.OptionGoOn, assign.OptionFailed} {
|
||||
if p.answeredWith(key, how) {
|
||||
w.Release, w.By, w.Why = how, "the operator", "the answer to the ask "+w.Ask
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// outsideApply keeps what a judging came to: the wait's record, its events, its finding and its ask.
|
||||
func (p *passer) outsideApply(ctx context.Context, key, hex, node, module, fp string, prev *outside.Wait, d outside.Decision, now time.Time) {
|
||||
o := p.outside
|
||||
switch {
|
||||
case d.End != "":
|
||||
p.outsideEnd(ctx, key, *prev, d.End, now)
|
||||
if d.Wait != nil {
|
||||
p.keepWait(ctx, key, *d.Wait)
|
||||
}
|
||||
return
|
||||
case d.Wait == nil:
|
||||
if prev != nil {
|
||||
p.outsideEnd(ctx, key, *prev, "", now)
|
||||
}
|
||||
return
|
||||
}
|
||||
w := *d.Wait
|
||||
if d.Held {
|
||||
body := p.outsideBody(w)
|
||||
if err := p.wantsSay(meshwants.EventWaitHeld, body, key+"."+w.On+"."+w.ClockFrom.UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
p.logf("the wait of %s on %s is not said, and is said at the next pass: %v", module, node, err)
|
||||
return
|
||||
}
|
||||
p.outsideFinding(w, true)
|
||||
}
|
||||
if d.Ask && w.Ask == "" {
|
||||
if id := p.outsideAsk(ctx, node, module, fp, w, now); id != "" {
|
||||
w.Ask = id
|
||||
}
|
||||
}
|
||||
if w.Standing() || w.Settling || w.Decided != "" {
|
||||
o.mu.Lock()
|
||||
if w.Cause != "" && w.Standing() {
|
||||
if !containsString(o.causes[w.Cause], node) {
|
||||
o.causes[w.Cause] = append(o.causes[w.Cause], node)
|
||||
}
|
||||
}
|
||||
o.mu.Unlock()
|
||||
if w.Settling {
|
||||
p.wakeAt(w.SettleEnds, now)
|
||||
} else if w.Standing() {
|
||||
p.wakeAt(w.ClockFrom.Add(o.bound), now)
|
||||
}
|
||||
p.keepWait(ctx, key, w)
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(l []string, s string) bool {
|
||||
for _, x := range l {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// keepWait writes the wait's record.
|
||||
func (p *passer) keepWait(ctx context.Context, key string, w outside.Wait) {
|
||||
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return outside.Encode(w) })
|
||||
}
|
||||
|
||||
func (p *passer) outsideBody(w outside.Wait) meshwants.Wait {
|
||||
since, from := w.Since, w.ClockFrom
|
||||
return meshwants.Wait{Step: meshwants.WaitStepOutside, Owner: assignment.Module(), Node: w.Node, Module: w.Module, Fingerprint: w.FP, On: w.On,
|
||||
Since: since, ClockFrom: &from, Bound: w.Bound,
|
||||
Words: fmt.Sprintf("%s on %s waits on %s", w.Module, w.Node, w.On)}
|
||||
}
|
||||
|
||||
// outsideEnd ends a wait: its wait-ended with how (none for a settle, which was never said), its finding and ask taken away, and
|
||||
// its record deleted unless the end decided it (the caller keeps that record).
|
||||
func (p *passer) outsideEnd(ctx context.Context, key string, w outside.Wait, how string, now time.Time) {
|
||||
if w.Standing() && how != "" {
|
||||
body := p.outsideBody(w)
|
||||
body.How = how
|
||||
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+"."+how+"."+now.UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
p.logf("the end of the wait of %s on %s is not said, and is said at the next pass: %v", w.Module, w.Node, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.outsideFinding(w, false)
|
||||
if ask, err := asks.OutsideAskKey(w.Node, w.Module, w.FP); err == nil {
|
||||
p.remove(ctx, buckets.Asks(), ask)
|
||||
}
|
||||
if p.outside != nil && w.Cause != "" {
|
||||
p.outside.mu.Lock()
|
||||
p.outside.causes[w.Cause] = removeString(p.outside.causes[w.Cause], w.Node)
|
||||
p.outside.mu.Unlock()
|
||||
}
|
||||
p.remove(ctx, buckets.Waits(), key)
|
||||
}
|
||||
|
||||
func removeString(l []string, s string) []string {
|
||||
var out []string
|
||||
for _, x := range l {
|
||||
if x != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// outsideFinding writes the finding delivery-waits-outside, or clears it.
|
||||
func (p *passer) outsideFinding(w outside.Wait, raised bool) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
o := assign.WaitsOutsideFinding(w.Node, w.Module, w.Cause, w.On, p.clock())
|
||||
if raised {
|
||||
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", "delivery."+o.ID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(mcseatKey(o)); err != nil {
|
||||
p.logf("the finding %s is not cleared: %v", "delivery."+o.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mcseatKey(o assign.Observed) string {
|
||||
return views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
|
||||
}
|
||||
|
||||
// outsideAsk asks the operator at the wait's bound, once, and returns the asking's id.
|
||||
func (p *passer) outsideAsk(ctx context.Context, node, module, fp string, w outside.Wait, now time.Time) string {
|
||||
key, err := asks.OutsideAskKey(node, module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
asking := assign.AskingSeconds(now)
|
||||
a := assign.OutsideAsk(key, asks.OutsideAbout(key), node, module, fp, w.On, w.ClockFrom, asking, now)
|
||||
p.put(ctx, buckets.Asks(), key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
|
||||
if ask, err := assign.AskOf(a); err == nil {
|
||||
return ask.ID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// wakeAt makes a pass when t comes: the settle's end and the bound are the waits' own times, the one wake (no polling).
|
||||
func (p *passer) wakeAt(t, now time.Time) {
|
||||
if p.outside == nil || p.outside.wake == nil {
|
||||
return
|
||||
}
|
||||
time.AfterFunc(max(t.Sub(now), 0)+time.Millisecond, p.outside.wake)
|
||||
}
|
||||
@@ -1,282 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/outside"
|
||||
)
|
||||
|
||||
// The whole of the wait on an outside cause in the pass (design 47 G §706): a fixture keeper answering `conditions` and saying
|
||||
// its events, a failing check of nats on ace, and the events, finding, ask and record the pass leaves.
|
||||
type outsideRig struct {
|
||||
t *testing.T
|
||||
p *passer
|
||||
now time.Time
|
||||
conds []map[string]any
|
||||
unread bool
|
||||
events []string
|
||||
bodies []meshwants.Wait
|
||||
found map[string]bool
|
||||
woke chan struct{}
|
||||
health string
|
||||
}
|
||||
|
||||
func newOutsideRig(t *testing.T) *outsideRig {
|
||||
r := &outsideRig{t: t, now: time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC), found: map[string]bool{}, woke: make(chan struct{}, 16), health: nodeengineclient.HealthFailed}
|
||||
r.p = newPasser(kvSet{buckets.Waits(): &memKV{values: map[string][]byte{}}, buckets.Asks(): &memKV{values: map[string][]byte{}}}, r.now)
|
||||
r.p.now = func() time.Time { return r.now }
|
||||
r.p.w.put(buckets.Asks(), "x", nil)
|
||||
r.p.outside = newOutsideActs(func(key string, body any) (json.RawMessage, error) {
|
||||
if r.unread {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
return json.Marshal(map[string]any{"conditions": r.conds})
|
||||
}, func() { r.woke <- struct{}{} }, func(string, ...any) {}, 30*time.Minute, 90*time.Second)
|
||||
r.p.publishWants = func(event string, body any, id string) error {
|
||||
r.events = append(r.events, event)
|
||||
r.bodies = append(r.bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
r.p.findings.Put = func(f mcseat.Finding) error { r.found[f.Scope+"."+f.ID+"."+f.Kind] = true; return nil }
|
||||
r.p.findings.Delete = func(key string) error { delete(r.found, key); return nil }
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *outsideRig) cond(key string, also ...string) {
|
||||
r.conds = append(r.conds, map[string]any{"key": key, "subject": map[string]any{"machine": strings.Split(key, ".")[1], "also": also}})
|
||||
}
|
||||
|
||||
// review is one pass's judging of nats on ace at the fingerprint, with the read answered at once when one is started.
|
||||
func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) assign.Nodes {
|
||||
r.t.Helper()
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
|
||||
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
|
||||
healthy := r.health == nodeengineclient.HealthOK
|
||||
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
|
||||
if moved == nil {
|
||||
moved = func(string, string) bool { return false }
|
||||
}
|
||||
out := r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
|
||||
select {
|
||||
case <-r.woke: // the read was answered: the next pass judges by it
|
||||
out = r.p.outsideReview(context.Background(), "nats", fpTest, []string{"ace"}, n, keeperSelf, moved)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *outsideRig) wait() outside.Wait {
|
||||
r.t.Helper()
|
||||
for _, v := range r.p.w.read(buckets.Waits()) {
|
||||
w, err := outside.Decode(v)
|
||||
if err != nil {
|
||||
r.t.Fatal(err)
|
||||
}
|
||||
return w
|
||||
}
|
||||
return outside.Wait{}
|
||||
}
|
||||
|
||||
func TestTheWaitOnAnOutsideCause(t *testing.T) {
|
||||
healthOf := func(n assign.Nodes) string { return n.Haves["ace"]["nats"].Health }
|
||||
t.Run("a cause listing the machine is a wait said at once, ended cleared when it clears healthy", func(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "ace")
|
||||
n := r.review(false, nil)
|
||||
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 1 || r.events[0] != meshwants.EventWaitHeld {
|
||||
t.Fatalf("health %s events %v", healthOf(n), r.events)
|
||||
}
|
||||
b := r.bodies[0]
|
||||
if b.Step != meshwants.WaitStepOutside || b.On != "unreachable:hub" || b.Bound != "30m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(r.now) || b.Node != "ace" || b.Module != "nats" {
|
||||
t.Errorf("wait-held %+v", b)
|
||||
}
|
||||
if !r.found["delivery.ace.nats.delivery-waits-outside"] {
|
||||
t.Errorf("findings %v", r.found)
|
||||
}
|
||||
// the cause clears and the module runs healthy
|
||||
r.conds, r.health, r.now = nil, nodeengineclient.HealthOK, r.now.Add(time.Minute)
|
||||
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-cleared", []byte(`{"key":"machine.hub.unreachable","event":"cleared","change":"cleared","at":"2026-11-11T10:01:00Z","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
|
||||
<-r.woke
|
||||
r.review(false, nil)
|
||||
if len(r.events) != 2 || r.events[1] != meshwants.EventWaitEnded || r.bodies[1].How != outside.HowCleared || len(r.found) != 0 || len(r.p.w.read(buckets.Waits())) != 0 {
|
||||
t.Errorf("events %v bodies %+v findings %v", r.events, r.bodies, r.found)
|
||||
}
|
||||
})
|
||||
t.Run("a cause not listing the machine settles, then the failure is the delivery's", func(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "g14")
|
||||
n := r.review(false, nil)
|
||||
if healthOf(n) == nodeengineclient.HealthFailed || len(r.events) != 0 || !r.wait().Settling {
|
||||
t.Fatalf("settle: health %s events %v wait %+v", healthOf(n), r.events, r.wait())
|
||||
}
|
||||
r.now = r.now.Add(91 * time.Second)
|
||||
if n := r.review(false, nil); healthOf(n) != nodeengineclient.HealthFailed {
|
||||
t.Errorf("past the settle the check still counts as failing: %s", healthOf(n))
|
||||
}
|
||||
})
|
||||
t.Run("a cause turning into another keeps the wait and its clock; a rewritten naming a moved module fails at once", func(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "ace")
|
||||
r.review(false, nil)
|
||||
r.conds = nil
|
||||
r.cond("machine.hub.network", "ace")
|
||||
r.now = r.now.Add(5 * time.Minute)
|
||||
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.network","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
|
||||
<-r.woke
|
||||
r.review(false, nil)
|
||||
last := len(r.bodies) - 1
|
||||
if len(r.events) != 2 || r.events[last] != meshwants.EventWaitHeld || r.bodies[last].On != "network:hub" || !r.bodies[last].ClockFrom.Equal(*r.bodies[0].ClockFrom) {
|
||||
t.Errorf("events %v bodies %+v", r.events, r.bodies)
|
||||
}
|
||||
r2 := newOutsideRig(t)
|
||||
r2.cond("machine.ace.nats.rewritten")
|
||||
n := r2.review(false, func(owner, node string) bool { return owner == "nats" })
|
||||
if healthOf(n) == nodeengineclient.HealthFailed || !r2.wait().Settling {
|
||||
t.Errorf("a rewritten of the moved module is its failure: %s %+v", healthOf(n), r2.wait())
|
||||
}
|
||||
})
|
||||
t.Run("the keeper silent waits on conditions-unread, and its own module is judged with no read", func(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.unread = true
|
||||
r.review(false, nil)
|
||||
r.now = r.now.Add(90 * time.Second) // the settle has run with no answer
|
||||
r.review(false, nil)
|
||||
if len(r.bodies) != 1 || r.bodies[0].On != outside.OnUnread {
|
||||
t.Fatalf("bodies %+v", r.bodies)
|
||||
}
|
||||
r2 := newOutsideRig(t)
|
||||
r2.cond("machine.hub.unreachable", "ace")
|
||||
if n := r2.review(true, nil); healthOf(n) != nodeengineclient.HealthFailed || len(r2.events) != 0 {
|
||||
t.Errorf("the keeper's own failing check waits: %s %v", healthOf(n), r2.events)
|
||||
}
|
||||
})
|
||||
t.Run("past the bound the operator is asked; Go on counts it healthy and Count as failed holds it, each said ended", func(t *testing.T) {
|
||||
for how, want := range map[string]string{assign.OptionGoOn: nodeengineclient.HealthOK, assign.OptionFailed: nodeengineclient.HealthFailed} {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.silent", "ace")
|
||||
r.review(false, nil)
|
||||
r.now = r.now.Add(31 * time.Minute)
|
||||
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-changed", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
|
||||
<-r.woke
|
||||
r.review(false, nil)
|
||||
key, _ := asks.OutsideAskKey("ace", "nats", fpTest)
|
||||
raw := r.p.w.value(buckets.Asks(), key)
|
||||
if raw == nil {
|
||||
t.Fatalf("%s: no ask at %s", how, key)
|
||||
}
|
||||
_, ask, err := assign.DecodeAsk(key, raw)
|
||||
if err != nil || ask.About != "ace.nats."+strings.TrimPrefix(key[strings.LastIndex(key, ".")+1:], "") || len(ask.Options) != 2 {
|
||||
t.Fatalf("ask %+v %v", ask, err)
|
||||
}
|
||||
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: how, Warrant: ask.ID + "@2026-11-11T10:32:00Z", Words: "the operator"})
|
||||
r.p.w.put(buckets.Answers, key, v)
|
||||
n := r.review(false, nil)
|
||||
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != how || healthOf(n) != want {
|
||||
t.Errorf("%s: events %v last %+v health %s", how, r.events, r.bodies[len(r.bodies)-1], healthOf(n))
|
||||
}
|
||||
if r.p.w.value(buckets.Asks(), key) != nil || len(r.found) != 0 {
|
||||
t.Errorf("%s: the ask or the finding stands after the end", how)
|
||||
}
|
||||
// a check that goes on failing is not waited on again
|
||||
r.now = r.now.Add(time.Minute)
|
||||
r.p.outside.Heard("mesh.seat.mesh-conditions.event.condition-raised", []byte(`{"key":"machine.hub.silent","subject":{"id":"hub","scope":"machine","machine":"hub","also":["ace"]}}`))
|
||||
<-r.woke
|
||||
before := len(r.events)
|
||||
r.review(false, nil)
|
||||
if len(r.events) != before {
|
||||
t.Errorf("%s: a decided check is waited on again", how)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("a newer want ends the wait replaced, and a restart judges it again", func(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "ace")
|
||||
r.review(false, nil)
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}}
|
||||
r.p.outsideReview(context.Background(), "nats", "src2:"+strings.Repeat("c", 64), []string{"ace"}, n, false, func(string, string) bool { return false })
|
||||
if r.events[len(r.events)-1] != meshwants.EventWaitEnded || r.bodies[len(r.bodies)-1].How != outside.HowReplaced {
|
||||
t.Errorf("events %v", r.events)
|
||||
}
|
||||
})
|
||||
t.Run("its bounds are read at the start from the files the manifest declares (A-112)", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, content string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
env := map[string]string{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": write("wait", "45\n"), "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": write("settle", "7")}
|
||||
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 45*time.Minute || settle != 7*time.Second {
|
||||
t.Errorf("bound %v settle %v err %v, want 45m and 7s", bound, settle, err)
|
||||
}
|
||||
env["MESH_ASSIGNER_OUTSIDE_WAIT_FILE"], env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = filepath.Join(dir, "missing"), write("empty", "")
|
||||
if bound, settle, err := outsideSettings(func(k string) string { return env[k] }); err != nil || bound != 0 || settle != 0 {
|
||||
t.Errorf("bound %v settle %v err %v, want both defaults", bound, settle, err)
|
||||
}
|
||||
env["MESH_ASSIGNER_OUTSIDE_SETTLE_FILE"] = write("bad", "soon")
|
||||
if _, _, err := outsideSettings(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") {
|
||||
t.Errorf("err %v, want one naming the file", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
|
||||
r := newOutsideRig(t)
|
||||
r.cond("machine.hub.unreachable", "ace")
|
||||
r.review(false, nil)
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
for k, v := range r.p.w.read(buckets.Waits()) {
|
||||
s.st.put(waitsState, k, v)
|
||||
}
|
||||
got, err := s.doWaits(map[string]any{"module": "nats"})
|
||||
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
|
||||
t.Errorf("wait = %+v", w)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("an unknown how is taken")
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on"}, "shanks.node-tools"); err == nil || len(acts) != 0 {
|
||||
t.Error("a release with no why is taken")
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "go-on", "why": "the hub is being rebuilt"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(acts) == 0 || !strings.Contains(acts[0], "hand-act") {
|
||||
t.Errorf("the hand act is not recorded first: %v", acts)
|
||||
}
|
||||
keys, _ := s.st.keys(waitsState)
|
||||
raw, _, _, _ := s.st.get(waitsState, keys[0])
|
||||
if w, _ := outside.Decode(raw); w.Release != "go-on" || !strings.Contains(w.By, "the hub is being rebuilt") {
|
||||
t.Errorf("wait = %+v", w)
|
||||
}
|
||||
// the process takes the release at its next pass: ended go-on, counted healthy
|
||||
r.p.w.put(buckets.Waits(), keys[0], raw)
|
||||
n := r.review(false, nil)
|
||||
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
|
||||
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/outside"
|
||||
)
|
||||
|
||||
// The tools of the waits on an outside cause (design 47 G §706, §276's row *delivery on an outside cause*): `assigner_waits` reads
|
||||
// each, the settle included, and `assigner_release` with `how` takes one. The tool writes the release into the wait's record and the
|
||||
// assigner's process ends the wait at its next pass, saying wait-ended with that how.
|
||||
|
||||
// toolWaits is the tool that reads the outside waits.
|
||||
const toolWaits = "assigner_waits"
|
||||
|
||||
// waitsState is the assigner's state of waits, as the runtime names a module's own state.
|
||||
const waitsState = buckets.WaitsState
|
||||
|
||||
// waitSaid is one wait as assigner_waits answers it: its record and where it stands.
|
||||
type waitSaid struct {
|
||||
Key string `json:"key"`
|
||||
outside.Wait
|
||||
// Settling's end is in the record; Ends is when the bound runs from clock-from, empty for a settle.
|
||||
BoundEnds string `json:"bound-ends,omitempty"`
|
||||
}
|
||||
|
||||
// doWaits is the tool assigner_waits.
|
||||
func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(waitsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []any{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(waitsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
w, err := outside.Decode(raw)
|
||||
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
said := waitSaid{Key: key, Wait: w}
|
||||
if w.Standing() {
|
||||
if d, err := time.ParseDuration(w.Bound); err == nil {
|
||||
said.BoundEnds = w.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
return map[string]any{"waits": out}, nil
|
||||
}
|
||||
|
||||
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
|
||||
// module, which the assigner's process takes at its next pass.
|
||||
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
|
||||
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
|
||||
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
|
||||
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept in the hand-act log")
|
||||
}
|
||||
keys, err := s.st.keys(waitsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
|
||||
}
|
||||
prefix := node + "." + module + "."
|
||||
var key string
|
||||
var w outside.Wait
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(waitsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if c, err := outside.Decode(raw); err == nil && c.Decided == "" {
|
||||
key, w = k, c
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("%s holds no wait of %s on an outside cause: assigner_waits lists them", node, module)
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, how), Why: why,
|
||||
Cause: "outside-wait", Verb: "mesh-wants.assigner_release"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
|
||||
}
|
||||
w.Release, w.By, w.Why = how, by+": "+why, why
|
||||
data, err := outside.Encode(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(waitsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// waitsTool is assigner_waits as the runtime serves it.
|
||||
func (s *putTools) waitsTool() stdio.Tool {
|
||||
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: a moved module's failing health check on a " +
|
||||
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
|
||||
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
|
||||
"machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil), Run: s.doWaits}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
package main
|
||||
|
||||
import plc "mesh-delivery-assigner/internal/assignment"
|
||||
|
||||
// The assigner's tests run as the module its manifest names, as a credential's module would say it at its start (design 47 G A-296).
|
||||
func init() { plc.SetModule("mesh-delivery-assigner") }
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,391 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The wants follow a passed pipeline (design 47 H §750): the runner says each pipeline's end once, per source revision, as
|
||||
// pipeline-ended on `pipeline-ended.<branch>.<source>`, naming per mover its fingerprint, its released version and its tier
|
||||
// (§746). The assigner keeps, in its state `written` (G §352), one key per module and watched branch: the newest released
|
||||
// version of the module on that branch whose pipeline passed, with the pipeline and the mover's rank in it. A newer version
|
||||
// replaces it by compare-and-set; an older one, the same one again, a failed pipeline and a run on request write nothing.
|
||||
|
||||
// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it.
|
||||
const (
|
||||
writtenState = "written"
|
||||
writtenTool = "assigner_written"
|
||||
)
|
||||
|
||||
// pipelineEndedSubject is the runner's event the assigner consumes.
|
||||
const pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>"
|
||||
|
||||
// End is one value of the state written: the newest version of a module released on a watched branch whose pipeline passed.
|
||||
type End struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Source int64 `json:"source"`
|
||||
Fp string `json:"fp"`
|
||||
Version string `json:"version"`
|
||||
// Rank is the mover's tier in the pipeline's delivery order, from 1 (§746, §750).
|
||||
Rank int64 `json:"rank"`
|
||||
}
|
||||
|
||||
// endKey is the key of a module on a watched branch, each name escaped as a key's name is.
|
||||
func endKey(module, branch string) string {
|
||||
return keys.EscapeName(module) + "." + keys.EscapeName(branch)
|
||||
}
|
||||
|
||||
// moverPassed says whether no required step of a mover stands failed, unasked or unended: a required step ending "nothing to
|
||||
// run" counts as passed, as a bypassed one does (design 47 H §746, H-A-3); a failed optional step holds nothing (G §30).
|
||||
func moverPassed(m mpr.MoverEnded) bool {
|
||||
for _, s := range m.Steps {
|
||||
if s.Status != mpr.StepResultStatusRequired {
|
||||
continue
|
||||
}
|
||||
switch s.Outcome {
|
||||
case mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeCouldNotRun, mpr.StepResultOutcomeDeclined, mpr.StepResultOutcomeNotAsked,
|
||||
mpr.StepResultOutcomeCancelled, mpr.StepResultOutcomeSuperseded, mpr.StepResultOutcomeDidNotBuild,
|
||||
mpr.StepResultOutcomeRunning, mpr.StepResultOutcomeWaiting, mpr.StepResultOutcomeWaitingForABuildNode:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// endStore is the kept state written: the bus bucket, or a test's.
|
||||
type endStore interface {
|
||||
// get is a key's value and revision; none for a key not there.
|
||||
get(key string) (value []byte, revision uint64, err error)
|
||||
// swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not.
|
||||
swap(key string, value []byte, revision uint64) error
|
||||
// keys are the keys that stand, and drop deletes one.
|
||||
keys() ([]string, error)
|
||||
drop(key string) error
|
||||
}
|
||||
|
||||
var errSwapped = errors.New("the key changed")
|
||||
|
||||
// kvEnds is endStore over a bus bucket.
|
||||
type kvEnds struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (k kvEnds) get(key string) ([]byte, uint64, error) {
|
||||
e, err := k.kv.Get(k.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (k kvEnds) swap(key string, value []byte, revision uint64) error {
|
||||
var err error
|
||||
if revision == 0 {
|
||||
_, err = k.kv.Create(k.ctx, key, value)
|
||||
} else {
|
||||
_, err = k.kv.Update(k.ctx, key, value, revision)
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || (err != nil && isWrongSequence(err)) {
|
||||
return errSwapped
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (k kvEnds) keys() ([]string, error) {
|
||||
ks, err := k.kv.Keys(k.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return ks, err
|
||||
}
|
||||
|
||||
func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) }
|
||||
|
||||
func isWrongSequence(err error) bool {
|
||||
var ae *jetstream.APIError
|
||||
return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
|
||||
}
|
||||
|
||||
// pipelineEnds follows the runner's ends; a nil *pipelineEnds keeps none, and no want is written from it.
|
||||
type pipelineEnds struct {
|
||||
st endStore
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// on says the first registration was heard (design 47 G §353), said once in the journal.
|
||||
on atomic.Bool
|
||||
mu sync.Mutex
|
||||
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
|
||||
// 47 G §14).
|
||||
firsts map[string]string
|
||||
// runEnded is told a run on request's pipeline-ended, its group `run-<id>`, to drop the run's candidate (A-235, H-A-58); nil
|
||||
// drops nothing.
|
||||
runEnded func(group string) error
|
||||
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
|
||||
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
|
||||
left atomic.Pointer[func(pipeline string) error]
|
||||
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
|
||||
onBranches func([]string)
|
||||
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
|
||||
trunk func() string
|
||||
// now is the clock of the tier progress (tiers.go); nil is time.Now.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
|
||||
return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}}
|
||||
}
|
||||
|
||||
// take keeps, for each mover of a passed pipeline that released a version, that version as the newest of its module on the
|
||||
// pipeline's branch, by compare-and-set on a newer version (design 47 H §750): an older version, or the same one again, writes
|
||||
// nothing. A passed mover with no version yet is said in the journal and taken when the runner says the end again naming it
|
||||
// (H-A-270); it holds its tier in the pipeline's tier progress meanwhile (H-A-277 (1)). It says whether any key changed.
|
||||
func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
}
|
||||
// The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not
|
||||
// know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress.
|
||||
if _, err := pipelinekey.Parse(p.Pipeline); err != nil {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
|
||||
return false, nil
|
||||
}
|
||||
changed := false
|
||||
var kept []tierMover
|
||||
versioned := map[string]bool{}
|
||||
defer func() {
|
||||
if err := e.startTiers(p.Pipeline, p.Source, kept, versioned); err != nil {
|
||||
e.logf("the tier progress of %s is not written: %v", p.Pipeline, err)
|
||||
}
|
||||
}()
|
||||
for _, m := range p.Movers {
|
||||
if m.Fingerprint == "" || !moverPassed(m) {
|
||||
continue
|
||||
}
|
||||
if m.Version == "" {
|
||||
// The runner writes a mover's version at its release's version-released, and says this end again once it does
|
||||
// (H-A-270 (1), (2)): the skip is said, never silent.
|
||||
e.logf("%s: %s passed with no version yet: taken again when its version is written", p.Pipeline, m.Module)
|
||||
// It holds its tier in the progress until its version is written (H-A-277 (1)).
|
||||
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1), NoVersion: true})
|
||||
continue
|
||||
}
|
||||
versioned[m.Module] = true
|
||||
key := endKey(m.Module, p.Branch)
|
||||
for {
|
||||
raw, rev, err := e.st.get(key)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if raw != nil {
|
||||
var kept End
|
||||
if err := json.Unmarshal(raw, &kept); err == nil && compareVersion(m.Version, kept.Version) <= 0 {
|
||||
break // an older version, or this one again
|
||||
}
|
||||
}
|
||||
value, err := json.Marshal(End{Pipeline: p.Pipeline, Source: p.Source, Fp: m.Fingerprint, Version: m.Version, Rank: max(m.Tier, 1)})
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
changed = true
|
||||
kept = append(kept, tierMover{Module: m.Module, Fp: m.Fingerprint, Tier: max(m.Tier, 1)})
|
||||
break
|
||||
}
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// endOf is the end kept for module on branch: the newest version released there whose pipeline passed.
|
||||
func (e *pipelineEnds) endOf(module, branch string) (End, bool) {
|
||||
if e == nil {
|
||||
return End{}, false
|
||||
}
|
||||
raw, _, err := e.st.get(endKey(module, branch))
|
||||
var kept End
|
||||
if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp == "" {
|
||||
return End{}, false
|
||||
}
|
||||
return kept, true
|
||||
}
|
||||
|
||||
// target is the build module's wants on branch name (design 47 H §750): the newest version released there whose pipeline
|
||||
// passed, with the pipeline and the mover's rank; false when none is kept, or its pipeline's tier progress has not reached its
|
||||
// rank (§751).
|
||||
func (e *pipelineEnds) target(module, branch string) (assign.Target, bool) {
|
||||
end, ok := e.endOf(module, e.branchOf(module, branch))
|
||||
if !ok || !e.tierWritten(end) {
|
||||
return assign.Target{}, false
|
||||
}
|
||||
return assign.Target{Fp: end.Fp, Version: end.Version, Pipeline: end.Pipeline, Rank: end.Rank}, true
|
||||
}
|
||||
|
||||
// hearPipeline takes a pipeline-ended: a passed one's released versions are kept (take), and a run on request's drops the
|
||||
// run's candidate assignment graph (A-235, H-A-58); a run delivers nothing (design 47 G §29, §64). A pipeline ended superseded or
|
||||
// cancelled has its composition steps leave the runs they read (G §723).
|
||||
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
|
||||
p, err := mprc.PipelineEnded(m.Data())
|
||||
if err != nil {
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if left := e.left.Load(); left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
|
||||
if err := (*left)(p.Pipeline); err != nil {
|
||||
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.Run != "" {
|
||||
if e.runEnded != nil {
|
||||
if err := e.runEnded(runGroupPrefix + p.Run); err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
changed, err := e.take(p)
|
||||
if err != nil {
|
||||
e.logf("%s: %v; it is heard again", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if changed {
|
||||
e.wake()
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// hearEnds registers pipeline-ended on the assigner's hub.
|
||||
func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
|
||||
if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil {
|
||||
return fmt.Errorf("%s: %w", pipelineEndedSubject, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's.
|
||||
func (e *pipelineEnds) branchOf(module, placed string) string {
|
||||
if placed != "" {
|
||||
return placed
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if b := e.firsts[module]; b != "" {
|
||||
return b
|
||||
}
|
||||
if e.trunk != nil {
|
||||
return e.trunk()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// writtenTools is the tool assigner_written: every key of the state written, with its end.
|
||||
func writtenTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the newest version released whose pipeline passed, with its pipeline and rank; per pipeline whose wants are written tier by tier, its tier progress (tier, since, movers); with why, why each want is what it is (as assigner_why).",
|
||||
Input: schema(map[string]string{"why": "true: why each want is what it is", "module": "with why: a module's name", "node": "with why: a node's name"}, nil, "why"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if args["why"] == true || args["why"] == "true" {
|
||||
return whyTools(st)[0].Run(args) // a want's why (why.go), until the manifest naming assigner_why is taken in
|
||||
}
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(ks)
|
||||
out := map[string]any{}
|
||||
for _, k := range ks {
|
||||
if strings.HasPrefix(k, whyPrefix) {
|
||||
continue // a want's why (why.go), read by assigner_why
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
if isTierKey(k) { // a pipeline's tier progress (tiers.go)
|
||||
var t tierProgress
|
||||
if json.Unmarshal(raw, &t) == nil {
|
||||
out[k] = t
|
||||
}
|
||||
continue
|
||||
}
|
||||
var e End
|
||||
if json.Unmarshal(raw, &e) == nil {
|
||||
out[k] = e
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
|
||||
// following says whether the wants follow the pipeline ends yet; a nil follower never does.
|
||||
func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() }
|
||||
|
||||
// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first
|
||||
// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone
|
||||
// is a registration removed: all its ends go.
|
||||
func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) {
|
||||
e.mu.Lock()
|
||||
if gone || len(reg.Branches) == 0 {
|
||||
delete(e.firsts, reg.Module)
|
||||
} else {
|
||||
e.firsts[reg.Module] = reg.Branches[0]
|
||||
}
|
||||
e.mu.Unlock()
|
||||
kept, err := e.st.keys()
|
||||
if err != nil {
|
||||
e.logf("the ends kept cannot be listed: %v", err)
|
||||
}
|
||||
prefix := keys.EscapeName(reg.Module) + "."
|
||||
for _, k := range kept {
|
||||
rest, ok := strings.CutPrefix(k, prefix)
|
||||
if !ok || isTierKey(k) || strings.HasPrefix(k, whyPrefix) {
|
||||
continue
|
||||
}
|
||||
branch, err := keys.UnescapeName(rest)
|
||||
if err == nil && !gone && slices.Contains(reg.Branches, branch) {
|
||||
continue
|
||||
}
|
||||
if err := e.st.drop(k); err != nil {
|
||||
e.logf("the end %s is not deleted: %v", k, err)
|
||||
}
|
||||
}
|
||||
if e.onBranches != nil && !gone {
|
||||
e.onBranches(reg.Branches)
|
||||
}
|
||||
if !e.on.Swap(true) {
|
||||
e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module)
|
||||
}
|
||||
e.wake()
|
||||
}
|
||||
@@ -1,306 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// memEnds is endStore over a map, with revisions.
|
||||
type memEnds struct {
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
}
|
||||
|
||||
func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil }
|
||||
func (m *memEnds) keys() (out []string, _ error) {
|
||||
for k := range m.vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil }
|
||||
func (m *memEnds) swap(key string, v []byte, rev uint64) error {
|
||||
if m.revs[key] != rev {
|
||||
return errSwapped
|
||||
}
|
||||
m.vals[key], m.revs[key] = v, rev+1
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back.
|
||||
type heardMsg struct {
|
||||
jetstream.Msg
|
||||
data []byte
|
||||
subject string
|
||||
acked, naked bool
|
||||
}
|
||||
|
||||
func (m *heardMsg) Data() []byte { return m.data }
|
||||
func (m *heardMsg) Subject() string { return m.subject }
|
||||
func (m *heardMsg) Ack() error { m.acked = true; return nil }
|
||||
func (m *heardMsg) Nak() error { m.naked = true; return nil }
|
||||
|
||||
// mover is one mover at its pipeline's end, its required unit step ending outcome.
|
||||
func mover(module, fp, version string, tier int64, outcome string) mpr.MoverEnded {
|
||||
return mpr.MoverEnded{Module: module, Fingerprint: fp, Version: version, Tier: tier,
|
||||
Steps: []mpr.StepResult{{Step: "unit", Status: mpr.StepResultStatusRequired, Outcome: outcome, Module: module}}}
|
||||
}
|
||||
|
||||
// ended is the pipeline-ended of the take-in of source on branch, its movers given.
|
||||
func ended(branch string, source int64, outcome string, movers ...mpr.MoverEnded) mpr.PipelineEnded {
|
||||
return mpr.PipelineEnded{V: 1, Branch: branch, Source: source, EndCount: 1, Outcome: outcome, Movers: movers,
|
||||
Pipeline: fmt.Sprintf("%s.%d", branch, source)}
|
||||
}
|
||||
|
||||
// A6/AC1 to AC3 (design 47 H §750, H-A-3): a passed pipeline keeps each mover's released version as the newest of its module on
|
||||
// the branch, with the pipeline and the mover's tier; a failed pipeline, a mover whose required step failed, one that released
|
||||
// no version, an older version, the same one again and a run on request keep nothing.
|
||||
func TestThePassedVersionIsKept(t *testing.T) {
|
||||
pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed
|
||||
ok, failed, nothing := mpr.StepResultOutcomePassed, mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeNothingToRun
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ends []mpr.PipelineEnded
|
||||
want *End // nil: none kept for web on main
|
||||
}{
|
||||
{"a passed pipeline keeps its version", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"nothing to run counts as passed", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, nothing))},
|
||||
&End{Pipeline: "main.5", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 1}},
|
||||
{"a failed pipeline keeps nothing, a passed mover in it neither", []mpr.PipelineEnded{ended("main", 5, fail, mover("web", "fp1", "0.1.2", 1, ok),
|
||||
mover("db", "fp9", "0.3.0", 1, failed))}, nil},
|
||||
{"a mover whose required step failed keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, failed))}, nil},
|
||||
{"no version released keeps nothing", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "", 1, ok))}, nil},
|
||||
{"an older end heard after a newer one keeps the newer", []mpr.PipelineEnded{ended("main", 6, pass, mover("web", "fp2", "0.1.3", 1, ok)),
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
|
||||
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.again.2"
|
||||
return p
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.later.2"
|
||||
return p
|
||||
}()}, nil},
|
||||
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Run = "r1"
|
||||
return p
|
||||
}()}, nil},
|
||||
} {
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
for _, p := range c.ends {
|
||||
if _, err := e.take(p); err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
got, kept := e.endOf("web", "main")
|
||||
if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 {
|
||||
t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks)
|
||||
}
|
||||
switch {
|
||||
case c.want == nil && kept:
|
||||
t.Errorf("%s: kept %+v", c.name, got)
|
||||
case c.want != nil && (!kept || got != *c.want):
|
||||
t.Errorf("%s: kept %+v (%v), want %+v", c.name, got, kept, *c.want)
|
||||
}
|
||||
}
|
||||
// The same end heard again changes nothing and wakes nothing; a run on request's end drops its candidate.
|
||||
woken := 0
|
||||
e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf)
|
||||
var dropped []string
|
||||
e.runEnded = func(group string) error { dropped = append(dropped, group); return nil }
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
for range 2 {
|
||||
raw, _ := json.Marshal(p)
|
||||
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"}
|
||||
e.hearPipeline(m)
|
||||
if !m.acked {
|
||||
t.Error("the end was not acknowledged")
|
||||
}
|
||||
}
|
||||
run := p
|
||||
run.Run = "r1"
|
||||
raw, _ := json.Marshal(run)
|
||||
e.hearPipeline(&heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.main.5"})
|
||||
if woken != 1 || !slices.Equal(dropped, []string{runGroupPrefix + "r1"}) {
|
||||
t.Errorf("woken %d (want 1), dropped %v (want the run's candidate)", woken, dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// A6/AC1, A5/AC1 (design 47 H §750): a pass writes, on every node the module is assigned to, the want naming the newest version
|
||||
// released on its branch whose pipeline passed, with the pipeline and the rank; with none passed it writes none. No grant,
|
||||
// canary, provider or composition is asked for. H-A-270 (1): a passed end whose mover names no version yet writes no want and
|
||||
// says so in the journal; the same end said again naming the version writes it.
|
||||
func TestAPassWritesTheWantOfThePassedVersion(t *testing.T) {
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, passed := range []bool{false, true} {
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web", Requires: []string{"database"}}}
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
var journal []string
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
p.ends.trunk = p.w.trunkName
|
||||
if passed {
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "", 2, mpr.StepResultOutcomePassed)))
|
||||
p.wants(context.Background())
|
||||
if len(wantsKV.values) != 0 || !slices.Contains(journal, trunkBranch+".5: web passed with no version yet: taken again when its version is written") {
|
||||
t.Errorf("an end naming no version wrote %d wants and said %q", len(wantsKV.values), journal)
|
||||
}
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 2, mpr.StepResultOutcomePassed)))
|
||||
}
|
||||
p.wants(context.Background())
|
||||
for _, node := range []string{"g14", "ace"} {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
raw, got := wantsKV.values[key]
|
||||
if got != passed {
|
||||
t.Errorf("passed %v: a want on %s written = %v", passed, node, got)
|
||||
continue
|
||||
}
|
||||
if !passed {
|
||||
continue
|
||||
}
|
||||
if w, err := mwc.DecodeWant(raw); err != nil || w.Fp != fp || w.Version != "0.1.2" || w.Pipeline != trunkBranch+".5" || w.Rank != 2 {
|
||||
t.Errorf("the want on %s = %+v (%v)", node, w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §353, §352: a registration's first branch is a placement's default, and a branch it no longer watches loses its
|
||||
// end; a registration gone loses them all.
|
||||
func TestARegistrationNamesTheDefaultBranch(t *testing.T) {
|
||||
st := newMemEnds()
|
||||
woken := 0
|
||||
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
|
||||
e.trunk = func() string { return trunkBranch }
|
||||
pass := mpr.PipelineEndedOutcomePassed
|
||||
_, _ = e.take(ended("main", 5, pass, mover("web", "fp1", "0.1.1", 1, mpr.StepResultOutcomePassed)))
|
||||
_, _ = e.take(ended("beta", 5, pass, mover("web", "fp2", "0.1.2", 1, mpr.StepResultOutcomePassed)))
|
||||
e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false)
|
||||
endsKept := func() (n int) { // a pipeline's tier progress is not a module's end
|
||||
for k := range st.vals {
|
||||
if !isTierKey(k) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if woken == 0 || endsKept() != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch {
|
||||
t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", endsKept(), e.branchOf("web", ""))
|
||||
}
|
||||
if tg, ok := e.target("web", ""); !ok || tg.Fp != "fp2" {
|
||||
t.Errorf("the default branch's target = %+v %v", tg, ok)
|
||||
}
|
||||
e.registered(mpcat.Registration{Module: "web"}, true)
|
||||
if endsKept() != 0 {
|
||||
t.Errorf("a registration gone left %d ends", endsKept())
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §14, §184 (test 68): a node's want follows the branch its placement names, so a version passed on `beta` moves only
|
||||
// the want of the node whose placement chose `beta`.
|
||||
func TestAVersionOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
|
||||
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
|
||||
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
|
||||
values, err := mmac.AssignmentsValues(mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(plc.AssignmentsBucket(), key, raw)
|
||||
}
|
||||
}
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp("a"), Manifest: strings.Repeat("e", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, "main", "web", mod)
|
||||
passed := func(branch, f, version string, source int64) {
|
||||
_, _ = p.ends.take(ended(branch, source, mpr.PipelineEndedOutcomePassed, mover("web", f, version, 1, mpr.StepResultOutcomePassed)))
|
||||
}
|
||||
fpWant := func(node string) string {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
w, err := mwc.DecodeWant(wantsKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatalf("the want of web on %s: %v", node, err)
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
passed("main", fp("a"), "0.1.1", 5)
|
||||
passed("beta", fp("b"), "0.2.1", 5)
|
||||
p.wants(context.Background())
|
||||
if fpWant("g14") != fp("a") || fpWant("ace") != fp("b") {
|
||||
t.Fatalf("each node is wanted at its branch's build: g14 %s, ace %s", fpWant("g14"), fpWant("ace"))
|
||||
}
|
||||
passed("beta", fp("d"), "0.2.2", 6)
|
||||
p.wants(context.Background())
|
||||
if fpWant("g14") != fp("a") || fpWant("ace") != fp("d") {
|
||||
t.Errorf("a version on beta moves only ace: g14 %s (want main's), ace %s (want beta's new)", fpWant("g14"), fpWant("ace"))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §14, §195: the catalog half of each branch is taken into the world per branch, an entry as the value and graph node
|
||||
// the rules read, and a registration's branches are each followed.
|
||||
func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
|
||||
w := newWorld()
|
||||
woken := 0
|
||||
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
|
||||
entry := func(fp string) json.RawMessage {
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
|
||||
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
|
||||
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
|
||||
return raw
|
||||
}
|
||||
h.changed("main", "web", entry("src2:1"), false)
|
||||
h.changed("beta", "web", entry("src2:2"), false)
|
||||
if w.trunkOf("main")["web"].Fp2 != "src2:1" || w.trunkOf("beta")["web"].Fp2 != "src2:2" || woken != 2 {
|
||||
t.Errorf("main %+v, beta %+v, woken %d: each branch keeps its own entry", w.trunkOf("main")["web"], w.trunkOf("beta")["web"], woken)
|
||||
}
|
||||
v := w.trunkOf("beta")["web"]
|
||||
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
|
||||
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
|
||||
}
|
||||
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
|
||||
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
|
||||
t.Errorf("graph node %+v", n)
|
||||
}
|
||||
h.changed("beta", "web", nil, true)
|
||||
if _, ok := w.trunkOf("beta")["web"]; ok || w.trunkOf("main")["web"].Fp2 == "" {
|
||||
t.Error("a module taken out of beta's half leaves main's")
|
||||
}
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
|
||||
func orderFP(c string) string {
|
||||
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
|
||||
}
|
||||
|
||||
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
|
||||
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
|
||||
type orderMod struct {
|
||||
name string
|
||||
run, trunk string
|
||||
forms map[string]int
|
||||
holds []string
|
||||
}
|
||||
|
||||
type orderFixture struct {
|
||||
t *testing.T
|
||||
p *passer
|
||||
wants *memKV
|
||||
events []string
|
||||
bodies []meshwants.Wait
|
||||
answers chan struct{}
|
||||
}
|
||||
|
||||
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
|
||||
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
|
||||
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
|
||||
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
|
||||
f.p = newPasser(kvSet{buckets.Wants(): f.wants, buckets.Asks(): &memKV{values: map[string][]byte{}}, buckets.Waits(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p := f.p
|
||||
p.logf = t.Logf
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
f.events = append(f.events, event)
|
||||
f.bodies = append(f.bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
pl := mma.Assignments{Node: "ace"}
|
||||
forms := map[string]int{} // by fingerprint: each module's own
|
||||
modules := []string{}
|
||||
for _, m := range mods {
|
||||
modules = append(modules, m.name)
|
||||
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
|
||||
for _, seat := range m.holds {
|
||||
scope := "node"
|
||||
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
|
||||
scope = "mesh"
|
||||
}
|
||||
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
|
||||
}
|
||||
for fp, form := range m.forms {
|
||||
forms[fp] = form
|
||||
}
|
||||
mdigest := fmt.Sprintf("%064x", len(modules))
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
endDone(t, p.w, m.name, m.trunk, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
f.runs(m.name, m.run)
|
||||
// The trunk's build is released with a passed pipeline (design 47 H §750).
|
||||
_, _ = p.ends.take(ended(trunkBranch, 1, mpr.PipelineEndedOutcomePassed, mover(m.name, m.trunk, "0.1.1", 1, mpr.StepResultOutcomePassed)))
|
||||
wkey, _ := mwc.WantKey("ace", m.name)
|
||||
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
|
||||
p.w.put(buckets.Wants(), wkey, old)
|
||||
f.wants.values[wkey] = old
|
||||
}
|
||||
composes(t, p.w, "ace", digest)
|
||||
values, err := mmac.AssignmentsValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(plc.AssignmentsBucket(), key, raw)
|
||||
}
|
||||
p.w.heardPresence("ace", false, time.Now()) // ace is online: its last presence event heard is machine-back
|
||||
objects := memObjects{}
|
||||
for i, at := range history {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, module := range modules {
|
||||
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
|
||||
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
|
||||
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
|
||||
}
|
||||
placed := mma.Assignments{Node: "ace"}
|
||||
for _, module := range modules {
|
||||
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
|
||||
}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]mma.Assignments{"ace": placed}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
|
||||
if form := forms[in.Fingerprint]; form > 0 {
|
||||
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
|
||||
return f
|
||||
}
|
||||
|
||||
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
|
||||
func (f *orderFixture) runs(module, fp string) {
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
f.p.w.put(buckets.Have, "ace."+module, have)
|
||||
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
|
||||
}
|
||||
|
||||
// pass makes a pass, and after it every catalog entry it asked is answered.
|
||||
func (f *orderFixture) pass() {
|
||||
f.t.Helper()
|
||||
f.p.wants(context.Background())
|
||||
asked := 0
|
||||
for _, row := range f.p.entries.Rows() {
|
||||
if row.State == meshcatalogclient.EntryAsked {
|
||||
asked++
|
||||
}
|
||||
}
|
||||
for range asked {
|
||||
select {
|
||||
case <-f.answers:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
f.t.Fatal("an entry asked was not answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wantAt is the fingerprint module is wanted at on ace.
|
||||
func (f *orderFixture) wantAt(module string) string {
|
||||
f.t.Helper()
|
||||
key, _ := mwc.WantKey("ace", module)
|
||||
w, err := mwc.DecodeWant(f.wants.values[key])
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
|
||||
func (f *orderFixture) waits() []string {
|
||||
var out []string
|
||||
for key := range f.p.w.read(buckets.Waits()) {
|
||||
out = append(out, key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A3/AC1 (design 47 H §771, §764): given a module whose kept data's form changes while the new versions of the node-backup holder and
|
||||
// of the bus module do not run, when the assigner passes, then the module is wanted at its new version, no `order.` key is kept and no
|
||||
// wait is said: the order on a machine is its node-engine's apply order, by rank.
|
||||
func TestNoWantWaitsForTheOrderOfADelivery(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
|
||||
})
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 0 {
|
||||
t.Fatalf("web at %s, waits %v, events %v: want web at its new version, no wait kept or said", got, f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
@@ -5,23 +5,15 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -41,12 +33,6 @@ const PutEvent = "put"
|
||||
type caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
|
||||
CallKept(key string, body any) (json.RawMessage, error)
|
||||
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
|
||||
CallFile(key string, body any) (json.RawMessage, error)
|
||||
// CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775).
|
||||
CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
@@ -62,40 +48,40 @@ type putActs struct {
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
|
||||
rec meshhandactsclient.Recorder
|
||||
rec handacts.Recorder
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{rec: meshhandactsclient.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
}
|
||||
|
||||
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||
func identity(key string, p puts.Put) string {
|
||||
func identity(key string, p delivery.Put) string {
|
||||
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// putsByModule is every put the assigner holds, by module then node.
|
||||
func (p *passer) putsByModule() map[string]map[string]puts.Put {
|
||||
out := map[string]map[string]puts.Put{}
|
||||
for key, value := range p.w.read(buckets.Puts()) {
|
||||
node, module, err := puts.ParseKey(key)
|
||||
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||
out := map[string]map[string]delivery.Put{}
|
||||
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
put, err := puts.Decode(value)
|
||||
put, err := delivery.DecodePut(value)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]puts.Put{}
|
||||
out[module] = map[string]delivery.Put{}
|
||||
}
|
||||
out[module][node] = put
|
||||
}
|
||||
@@ -108,12 +94,12 @@ func (p *passer) puts(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
all := p.putsByModule()
|
||||
defer func() { p.putWaits(ctx, p.putsByModule()) }()
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned := p.w.assigned()
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
@@ -123,26 +109,34 @@ func (p *passer) puts(ctx context.Context) {
|
||||
for _, module := range modules {
|
||||
var manifest catalogue.Manifest
|
||||
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||
if got := p.w.manifest(mod.Manifest); got.err == "" {
|
||||
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||
manifest = got.manifest
|
||||
}
|
||||
}
|
||||
for node, put := range all[module] {
|
||||
if puts.Ended(put.Step) {
|
||||
if delivery.PutEnded(put.Step) {
|
||||
continue
|
||||
}
|
||||
key, _ := puts.Key(node, module)
|
||||
key, _ := delivery.PutKey(node, module)
|
||||
id := identity(key, put)
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, puts.Refused, "the bus's want is not written: "+err.Error())}
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) &&
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||
!strings.Contains(next.Put.Words, failed) {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||
}
|
||||
@@ -151,67 +145,71 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if !written {
|
||||
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||
}
|
||||
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted {
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !mwc.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.putWant(ctx, node, module, want)
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
if next.Warranted {
|
||||
p.acts.warrant(id, meshdeliveryclient.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalWant(w mwc.Want) mwc.Want {
|
||||
func normalWant(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
// putWorld is what one step of a put reads from the world.
|
||||
func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].Fp2}
|
||||
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||
for _, m := range assignedHere {
|
||||
if m == module {
|
||||
w.Assigned = true
|
||||
}
|
||||
}
|
||||
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
|
||||
w.Build, w.BuildWords = e.Outcome, e.Error
|
||||
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||
w.Build, w.BuildWords = b.State, b.Error
|
||||
} else {
|
||||
w.Build = "unreadable: " + err.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
|
||||
if c, ok := chosen[module]; ok {
|
||||
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
|
||||
} else if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := asks.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
expires := assign.ExpiresAt(ask)
|
||||
w.AskExpired = !expires.IsZero() && w.Now.After(expires)
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if r, ok := p.w.runningOf(node, module); ok {
|
||||
w.Running = &r
|
||||
}
|
||||
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(buckets.Have, hkey); raw != nil {
|
||||
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
|
||||
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
w.Have = &h
|
||||
}
|
||||
}
|
||||
@@ -222,15 +220,15 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
|
||||
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
|
||||
var out *assign.ComposedDigest
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := grantsstate.PartGrantKey(module, fp); err == nil {
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := grantsstate.DecodeGrant(raw); err == nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
@@ -240,17 +238,17 @@ func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, b
|
||||
|
||||
// actOn does the acts a step asks for.
|
||||
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||
if next.Put.Step == puts.Person && next.Put.Ask != "" && w.Composed != nil {
|
||||
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks(), akey))
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -259,29 +257,29 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
|
||||
func (p *passer) performs(key string, act occ.Act) error {
|
||||
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks(), key))
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range ask.Options {
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return occ.Performs(o, act)
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
}
|
||||
|
||||
// samePut says two puts say the same: their step, words and what they found.
|
||||
func samePut(a, b puts.Put) bool {
|
||||
func samePut(a, b delivery.Put) bool {
|
||||
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||
}
|
||||
|
||||
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||
// released it since; and says it as the event put. It says whether it wrote.
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put) bool {
|
||||
kv := p.kvs[buckets.Puts()]
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||
if kv == nil {
|
||||
return false
|
||||
}
|
||||
@@ -290,11 +288,11 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
|
||||
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
now, err := puts.Decode(e.Value())
|
||||
now, err := delivery.DecodePut(e.Value())
|
||||
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||
return false // asked again or released meanwhile: the next pass reads that
|
||||
}
|
||||
data, err := puts.Encode(next)
|
||||
data, err := delivery.EncodePut(next)
|
||||
if err != nil {
|
||||
p.logf("the put %s is not written: %v", key, err)
|
||||
return false
|
||||
@@ -305,7 +303,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.put(buckets.Puts(), key, data)
|
||||
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
p.acts.say(key, next)
|
||||
return true
|
||||
@@ -401,7 +399,7 @@ func (a *putActs) warrant(id, ask string) {
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.rec.RecordWarrant(buckets.TellerModule, ask); err != nil {
|
||||
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
@@ -430,8 +428,8 @@ type putSaid struct {
|
||||
}
|
||||
|
||||
// say says a put's step as the event put, once per step and words.
|
||||
func (a *putActs) say(key string, p puts.Put) {
|
||||
node, module, err := puts.ParseKey(key)
|
||||
func (a *putActs) say(key string, p delivery.Put) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || a.bus == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its
|
||||
// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint
|
||||
// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release,
|
||||
// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none.
|
||||
|
||||
// putWaitBound is the bound of a put without hold's wait.
|
||||
const putWaitBound = 30 * time.Minute
|
||||
|
||||
// Wait-ended's how of a put's wait.
|
||||
const (
|
||||
putWaitRan = "ran"
|
||||
putWaitReleased = "released"
|
||||
putWaitSuperseded = "superseded"
|
||||
)
|
||||
|
||||
// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run.
|
||||
func putWaits(p puts.Put) bool {
|
||||
return !p.Hold && p.Step == puts.Wanted
|
||||
}
|
||||
|
||||
// putWaitBody is the wait of the put p of module on node.
|
||||
func putWaitBody(node, module string, p puts.Put) meshwants.Wait {
|
||||
since := p.StandsSince().UTC()
|
||||
return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module(), Node: node, Module: module, Fingerprint: p.FP,
|
||||
Since: since, ClockFrom: &since, Bound: putWaitBound.String(),
|
||||
Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)}
|
||||
}
|
||||
|
||||
// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204,
|
||||
// A-211): its identity and the put as it was then.
|
||||
type putWaiting struct {
|
||||
ID string `json:"id"`
|
||||
Put puts.Put `json:"put"`
|
||||
}
|
||||
|
||||
// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left.
|
||||
func (p *passer) saidPutWaits() map[string]putWaiting {
|
||||
out := map[string]putWaiting{}
|
||||
for key, value := range p.w.read(buckets.PutWaits()) {
|
||||
var w putWaiting
|
||||
if err := json.Unmarshal(value, &w); err != nil || w.ID == "" {
|
||||
p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits(), key, err)
|
||||
continue
|
||||
}
|
||||
out[key] = w
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended
|
||||
// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said;
|
||||
// one not said, or not kept, is said at the next pass. Without the state no put's wait is said.
|
||||
func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) {
|
||||
if p.kvs[buckets.PutWaits()] == nil {
|
||||
return
|
||||
}
|
||||
waiting := p.saidPutWaits()
|
||||
for key, was := range waiting {
|
||||
node, module, err := puts.ParseKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
cur, there := all[module][node]
|
||||
same := there && identity(key, cur) == was.ID
|
||||
if same && putWaits(cur) {
|
||||
continue
|
||||
}
|
||||
how := putWaitSuperseded
|
||||
if same && cur.Step == puts.Done {
|
||||
how = putWaitRan
|
||||
} else if same && cur.Step == puts.Released {
|
||||
how = putWaitReleased
|
||||
}
|
||||
body := putWaitBody(node, module, was.Put)
|
||||
body.How = how
|
||||
if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil {
|
||||
p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.remove(ctx, buckets.PutWaits(), key)
|
||||
delete(waiting, key)
|
||||
}
|
||||
for module, byNode := range all {
|
||||
for node, put := range byNode {
|
||||
key, err := puts.Key(node, module)
|
||||
if err != nil || !putWaits(put) {
|
||||
continue
|
||||
}
|
||||
id := identity(key, put)
|
||||
if w, said := waiting[key]; said && w.ID == id {
|
||||
continue
|
||||
}
|
||||
if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil {
|
||||
p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err)
|
||||
continue
|
||||
}
|
||||
kept := putWaiting{ID: id, Put: put}
|
||||
p.put(ctx, buckets.PutWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as
|
||||
// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is
|
||||
// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said
|
||||
// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one
|
||||
// that ran while the assigner was stopped gets its wait-ended at the next start.
|
||||
func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) {
|
||||
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
|
||||
kept := &memKV{values: map[string][]byte{}}
|
||||
var events []string
|
||||
var bodies []meshwants.Wait
|
||||
start := func() *passer { // a process's start: a fresh world that hands what the state holds
|
||||
p := newPasser(kvSet{buckets.PutWaits(): kept}, at)
|
||||
for k, v := range kept.values {
|
||||
p.w.put(buckets.PutWaits(), k, v)
|
||||
}
|
||||
p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {})
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
events, bodies = append(events, event), append(bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
return p
|
||||
}
|
||||
ctx := context.Background()
|
||||
asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked")
|
||||
wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed")
|
||||
held := wanted
|
||||
held.Hold = true
|
||||
p := start()
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
|
||||
if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil {
|
||||
t.Fatalf("a put that waits said %v, kept %q", events, kept.values)
|
||||
}
|
||||
events, bodies = nil, nil
|
||||
// At the next start the put still waits: it was said before, nothing is said of it.
|
||||
p = start()
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("a put waiting at the start was said again: %v", events)
|
||||
}
|
||||
// It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it.
|
||||
p = start()
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}})
|
||||
if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut ||
|
||||
kept.values["ace.nats"] != nil {
|
||||
t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values)
|
||||
}
|
||||
events, bodies = nil, nil
|
||||
// A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing.
|
||||
again := wanted
|
||||
again.At = at.Add(time.Hour)
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
|
||||
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
|
||||
t.Fatalf("a put's written want said %v", events)
|
||||
}
|
||||
b := bodies[0]
|
||||
if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil ||
|
||||
!b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" {
|
||||
t.Errorf("the put's wait is %+v", b)
|
||||
}
|
||||
// Said once.
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
|
||||
if len(events) != 1 {
|
||||
t.Errorf("a put's wait was said again: %v", events)
|
||||
}
|
||||
// Put again under its key: the first wait ends superseded, the new one is held.
|
||||
newer := asked
|
||||
newer.At = at.Add(2 * time.Hour)
|
||||
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}})
|
||||
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest {
|
||||
t.Errorf("a put asked again said %v %+v", events, bodies)
|
||||
}
|
||||
// The answer of assigner_puts.
|
||||
if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" {
|
||||
t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound)
|
||||
}
|
||||
if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" {
|
||||
t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound)
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
|
||||
type releaseHeard struct {
|
||||
Version string
|
||||
Parts map[string]string
|
||||
Source int64
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
|
||||
func presenceOf(data []byte) (string, time.Time, error) {
|
||||
var e wd.MachineAwayEvent // machine-back has the same fields
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
if e.Machine == "" {
|
||||
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
|
||||
}
|
||||
return e.Machine, e.At, nil
|
||||
}
|
||||
|
||||
// hearReleases reads whole, from their first message, then follows, the catalog seat's version-released and the watchman's
|
||||
// machine-away and machine-back (mesh-sdk process.FromFirst, A-233), never their state. A stream not heard is said as the assigner's finding, and what it carries is not known until
|
||||
// the assigner starts again.
|
||||
func hearReleases(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
|
||||
acct := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
nh := notHeardWriter(findings, logf)
|
||||
for _, s := range []struct {
|
||||
stream, words string
|
||||
take func([]byte) error
|
||||
}{
|
||||
{meshcatalog.StreamVersionReleased, "no want names a released version", w.released.put},
|
||||
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, true, at)
|
||||
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", m)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
|
||||
m, at, err := presenceOf(data)
|
||||
if err == nil {
|
||||
w.heardPresence(m, false, at)
|
||||
logf("%s is back (machine-back): a composition step composes it again", m)
|
||||
}
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
take := s.take
|
||||
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
|
||||
if err := take(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf, process.FromFirst())
|
||||
if err != nil {
|
||||
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
|
||||
continue
|
||||
}
|
||||
nh.Heard(s.stream)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
|
||||
var revertReasons = []string{"restore point missing", "restore point damaged"}
|
||||
|
||||
// revertReason is the reason of a have's words when they say a held revert, or "".
|
||||
func revertReason(words string) string {
|
||||
for _, r := range revertReasons {
|
||||
if strings.HasPrefix(words, r) {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
|
||||
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
|
||||
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
|
||||
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
|
||||
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
|
||||
// that holds a revert, which backups leaves to it.
|
||||
func (p *passer) reverts(ctx context.Context) map[string]bool {
|
||||
taken := map[string]bool{}
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(buckets.Have) {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
have, err := nodeengineclient.DecodeHave(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
}
|
||||
akey, err := asks.RevertAskKey(node, module, want.Fp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reason := revertReason(have.Detail)
|
||||
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
|
||||
if have.Fp == want.Fp && p.w.has(buckets.Asks(), akey) {
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
|
||||
}
|
||||
continue
|
||||
}
|
||||
taken[node+"."+module] = true
|
||||
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
|
||||
if there != nil {
|
||||
now := p.clock()
|
||||
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
|
||||
p.put(ctx, buckets.Asks(), akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
|
||||
continue
|
||||
}
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
|
||||
continue // not answered, or answered hold: the hold stands and nothing is written
|
||||
}
|
||||
words := assign.RevertAnswerWords(answer.Option, current.Asking)
|
||||
if want.Revert != mwc.RevertLive || want.Words != words {
|
||||
want.Revert, want.Words = mwc.RevertLive, words
|
||||
p.putWant(ctx, node, module, want)
|
||||
}
|
||||
}
|
||||
return taken
|
||||
}
|
||||
|
||||
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
|
||||
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
|
||||
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
|
||||
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
|
||||
if p.graph == nil || p.entries == nil {
|
||||
return "", "", false
|
||||
}
|
||||
run, heard := p.w.runningOf(node, module)
|
||||
if !heard || run.Stopped || run.Fingerprint == "" {
|
||||
return "", "", false
|
||||
}
|
||||
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
|
||||
if err != nil {
|
||||
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
|
||||
return "", "", false
|
||||
}
|
||||
if !reverted {
|
||||
return "", "", false
|
||||
}
|
||||
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
|
||||
if !known {
|
||||
return "", "", false // asked: its answer wakes a pass
|
||||
}
|
||||
var names, forms []string
|
||||
for _, c := range changes {
|
||||
f := 0
|
||||
if c.Applied != nil {
|
||||
f = c.Applied.Form
|
||||
}
|
||||
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return "", "", false
|
||||
}
|
||||
return strings.Join(names, ", "), strings.Join(forms, ", "), true
|
||||
}
|
||||
|
||||
// revertFinding writes the finding revert-held, or clears it, once per process each way.
|
||||
func (p *passer) revertFinding(o assign.Observed, raised bool) {
|
||||
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
|
||||
said := key + ".raised"
|
||||
if !raised {
|
||||
said = key + ".cleared"
|
||||
}
|
||||
if p.said[said] {
|
||||
return
|
||||
}
|
||||
var err error
|
||||
switch {
|
||||
case raised && p.findings.Put != nil:
|
||||
err = p.findings.Put(o.Finding(p.clock()))
|
||||
case !raised && p.findings.Delete != nil:
|
||||
err = p.findings.Delete(key)
|
||||
default:
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the finding %s is not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.said[said] = true
|
||||
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
|
||||
}
|
||||
|
||||
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
|
||||
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
|
||||
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
|
||||
if p.entries == nil {
|
||||
return nil, false
|
||||
}
|
||||
running, rs := p.entries.Need(ctx, runFp)
|
||||
applied, as := p.entries.Need(ctx, wantFp)
|
||||
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
|
||||
return nil, false
|
||||
}
|
||||
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
|
||||
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
|
||||
}
|
||||
changes, err := assigngraph.FormChanges(running, applied)
|
||||
if err != nil {
|
||||
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
|
||||
return nil, false
|
||||
}
|
||||
return changes, true
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
|
||||
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
|
||||
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
|
||||
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Asks(): asksKV, buckets.Wants(): wantsKV}, at)
|
||||
p.logf = t.Logf
|
||||
findings := map[string][]byte{}
|
||||
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
|
||||
|
||||
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
|
||||
objects := memObjects{}
|
||||
for i, fp := range []string{fpOld, fpTest} {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
|
||||
entry := func(fp string, form int) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
|
||||
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
|
||||
}
|
||||
answered := make(chan struct{}, 4)
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
|
||||
|
||||
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
|
||||
wkey, _ := mwc.WantKey("ace", "web")
|
||||
want := mwc.Want{Fp: fpOld, Self: true}
|
||||
raw, _ := mwc.EncodeWant(want)
|
||||
p.w.put(buckets.Wants(), wkey, raw)
|
||||
hkey, _ := nodeengineclient.HaveKey("ace", "web")
|
||||
have := func(fp, phase, detail string) {
|
||||
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Have, hkey, raw)
|
||||
}
|
||||
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
|
||||
pass := func() { p.backups(ctx, p.reverts(ctx)) }
|
||||
|
||||
pass() // the entries are asked; nothing is asked of the operator until they answer
|
||||
for range 2 {
|
||||
select {
|
||||
case <-answered:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
|
||||
}
|
||||
}
|
||||
pass()
|
||||
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
|
||||
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
|
||||
if !p.w.has(buckets.Asks(), rkey) || p.w.has(buckets.Asks(), bkey) {
|
||||
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
|
||||
}
|
||||
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks())[rkey])
|
||||
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
|
||||
t.Fatalf("the revert ask = %+v, %v", ask, err)
|
||||
}
|
||||
fkey := "delivery.ace.web.revert-held"
|
||||
if _, raised := findings[fkey]; !raised {
|
||||
t.Fatalf("findings %v: want %s", findings, fkey)
|
||||
}
|
||||
|
||||
answer := func(option string) {
|
||||
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
|
||||
p.w.put(buckets.Answers, rkey, v)
|
||||
}
|
||||
puts := wantsKV.puts
|
||||
answer(assign.OptionRevertHold)
|
||||
pass()
|
||||
if wantsKV.puts != puts {
|
||||
t.Fatal("an answer hold wrote a want")
|
||||
}
|
||||
answer(assign.OptionRevertLive)
|
||||
pass()
|
||||
got, err := mwc.DecodeWant(p.w.read(buckets.Wants())[wkey])
|
||||
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
|
||||
t.Fatalf("the want after live = %+v, %v", got, err)
|
||||
}
|
||||
|
||||
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
|
||||
pass()
|
||||
if _, raised := findings[fkey]; raised {
|
||||
t.Fatal("the finding stands while the reverted build runs")
|
||||
}
|
||||
|
||||
// A newer want replaces the held revert: the ask goes and the finding with it.
|
||||
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
|
||||
pass()
|
||||
if _, raised := findings[fkey]; !raised {
|
||||
t.Fatal("the finding is not raised again for a hold again")
|
||||
}
|
||||
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
|
||||
p.w.put(buckets.Wants(), wkey, raw)
|
||||
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
|
||||
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
|
||||
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
|
||||
}
|
||||
|
||||
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
|
||||
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
|
||||
for _, withWait := range []bool{true, false} {
|
||||
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
|
||||
if withWait {
|
||||
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
|
||||
}
|
||||
raw, err := nodeengineclient.EncodeHave(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Have, hkey, raw)
|
||||
pass()
|
||||
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
|
||||
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// The assignment graph revisions are removed when they are no longer current (design 47 H §764, H-A-86): the gate's pins of
|
||||
// them went with its pinned, freed, heads-replaced and pins-stated (H-A-48), and with them the assigner's state pins and its
|
||||
// tools assigner_pins and assigner_unpin. A revision that is not current (the newest the assigner wrote) is removed, its object
|
||||
// and the entry objects no kept revision or candidate lists and no want or running build names (G §179), when a revision ceases
|
||||
// to be current and once at the start. Never on a timer.
|
||||
|
||||
// revisionObjects is the object bucket of the revisions as the remover removes from it.
|
||||
type revisionObjects interface {
|
||||
assigngraph.Objects
|
||||
Delete(ctx context.Context, name string) error
|
||||
}
|
||||
|
||||
// revisionRemover removes the revisions that are not current.
|
||||
type revisionRemover struct {
|
||||
ctx context.Context
|
||||
objects revisionObjects
|
||||
// inUse says a want or a running build names the module's fingerprint (§179).
|
||||
inUse func(module, fp string) bool
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// judge removes every revision that is not current; a removal that fails is said and made at the next judging.
|
||||
func (k *revisionRemover) judge() {
|
||||
if err := k.remove(); err != nil {
|
||||
k.logf("the revisions that are not current are not removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// trunkRevision is the revision an object of a trunk revision names, and whether it names one.
|
||||
func trunkRevision(name string) (int64, bool) {
|
||||
n, ok := strings.CutPrefix(name, "trunk.")
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
r, err := strconv.ParseInt(n, 10, 64)
|
||||
return r, err == nil && r > 0
|
||||
}
|
||||
|
||||
// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or
|
||||
// running build names (design 47 G §179).
|
||||
func (k *revisionRemover) remove() error {
|
||||
names, described, err := k.listing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var kept []int64
|
||||
for _, n := range names {
|
||||
if r, ok := trunkRevision(n); ok {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return nil
|
||||
}
|
||||
current := slices.Max(kept)
|
||||
listed := map[string]bool{}
|
||||
removed := 0
|
||||
for _, r := range kept {
|
||||
if r == current {
|
||||
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
|
||||
return fmt.Errorf("revision %d: %w", r, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
if removed == 0 {
|
||||
return nil
|
||||
}
|
||||
// A run's candidate kept now lists its entries too, until its run's pipeline-ended drops it (A-201, A-235).
|
||||
for _, n := range names {
|
||||
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range rev.Entries {
|
||||
listed[e.Digest] = true
|
||||
}
|
||||
}
|
||||
for _, n := range names {
|
||||
digest, ok := strings.CutPrefix(n, "entry.")
|
||||
if !ok || listed[digest] {
|
||||
continue
|
||||
}
|
||||
e, err := k.entryOf(n, digest, described)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
|
||||
continue
|
||||
}
|
||||
if err := k.objects.Delete(k.ctx, n); err != nil {
|
||||
return fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
}
|
||||
k.logf("%d assignment graph revisions not current are removed", removed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
|
||||
func inUseOf(w *world) func(module, fp string) bool {
|
||||
return func(module, fp string) bool {
|
||||
for _, want := range w.wantsOf()[module] {
|
||||
if want.Fp == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, mods := range w.runningNow() {
|
||||
if r, ok := mods[module]; ok && r.Fingerprint == fp {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// listing is every object's name in one read and, where the bucket says them, each entry object's module and fingerprint by name
|
||||
// (design 47 H-A-259); nil described where the bucket says none.
|
||||
func (k *revisionRemover) listing() ([]string, map[string]assigngraph.Described, error) {
|
||||
d, ok := k.objects.(assigngraph.Describer)
|
||||
if !ok {
|
||||
names, err := k.objects.Names(k.ctx)
|
||||
return names, nil, err
|
||||
}
|
||||
listed, err := d.Described(k.ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
names := make([]string, 0, len(listed))
|
||||
described := make(map[string]assigngraph.Described, len(listed))
|
||||
for _, l := range listed {
|
||||
names = append(names, l.Name)
|
||||
described[l.Name] = l
|
||||
}
|
||||
return names, described, nil
|
||||
}
|
||||
|
||||
// entryOf is the module and fingerprint of the entry object name: as the listing says them, with no read; an entry put before the
|
||||
// listing said them is read once and put again with them, so no later judging reads it (design 47 H-A-259).
|
||||
func (k *revisionRemover) entryOf(name, digest string, described map[string]assigngraph.Described) (meshwants.AssignmentEntry, error) {
|
||||
if l, ok := described[name]; ok && l.Module != "" && l.Fp != "" {
|
||||
return meshwants.AssignmentEntry{Module: l.Module, Fp: l.Fp}, nil
|
||||
}
|
||||
data, err := k.objects.Get(k.ctx, name)
|
||||
if err != nil {
|
||||
return meshwants.AssignmentEntry{}, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
var e meshwants.AssignmentEntry
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return e, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
if described != nil {
|
||||
if err := assigngraph.PutEntry(k.ctx, k.objects, name, data, e); err != nil {
|
||||
k.logf("the entry %s is not put again with its module and fingerprint, so the next judging reads it again: %v", digest, err)
|
||||
}
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
|
||||
|
||||
// describedObjects is memObjects whose listing says each entry's module and fingerprint (assigngraph.Describer), counting the
|
||||
// entry objects read.
|
||||
type describedObjects struct {
|
||||
memObjects
|
||||
meta map[string][2]string
|
||||
reads int
|
||||
}
|
||||
|
||||
func (d *describedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if strings.HasPrefix(name, "entry.") {
|
||||
d.reads++
|
||||
}
|
||||
return d.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
func (d *describedObjects) PutDescribed(_ context.Context, name string, data []byte, module, fp string) error {
|
||||
d.memObjects[name], d.meta[name] = data, [2]string{module, fp}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *describedObjects) Described(context.Context) ([]assigngraph.Described, error) {
|
||||
var out []assigngraph.Described
|
||||
for n := range d.memObjects {
|
||||
out = append(out, assigngraph.Described{Name: n, Module: d.meta[n][0], Fp: d.meta[n][1]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the
|
||||
// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists,
|
||||
// a running build's entry and a run's kept candidate with its entries stay.
|
||||
// H-A-259: where the listing says each entry's module and fingerprint, the judging reads no entry object; an entry put before it
|
||||
// said them is read once and put again with them.
|
||||
func TestARevisionNotCurrentIsRemoved(t *testing.T) {
|
||||
for _, listed := range []string{"plain", "described", "described-later"} {
|
||||
objects := memObjects{}
|
||||
described := &describedObjects{memObjects: objects, meta: map[string][2]string{}}
|
||||
entry := func(module, fp string) string {
|
||||
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
|
||||
d := assigngraph.EntryDigest(e)
|
||||
raw, _ := json.Marshal(e)
|
||||
objects[assigngraph.EntryObject(d)] = raw
|
||||
if listed == "described" {
|
||||
described.meta[assigngraph.EntryObject(d)] = [2]string{module, fp}
|
||||
}
|
||||
return d
|
||||
}
|
||||
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
|
||||
revision := func(n int64, digests ...string) {
|
||||
rev := meshwants.AssignmentRevision{Revision: n}
|
||||
for _, d := range digests {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
|
||||
}
|
||||
raw, _ := json.Marshal(rev)
|
||||
objects[assigngraph.TrunkObject(n)] = raw
|
||||
}
|
||||
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
|
||||
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
|
||||
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
|
||||
revision(1, shared, old, running)
|
||||
revision(2, shared, old)
|
||||
revision(3, shared)
|
||||
var o revisionObjects = objects
|
||||
if listed != "plain" {
|
||||
o = described
|
||||
}
|
||||
k := &revisionRemover{ctx: context.Background(), objects: o, logf: t.Logf,
|
||||
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
|
||||
k.judge()
|
||||
has := func(name string) bool { _, ok := objects[name]; return ok }
|
||||
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
|
||||
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
|
||||
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
|
||||
if has(name) != want {
|
||||
t.Errorf("%s: %s kept = %v, want %v", listed, name, has(name), want)
|
||||
}
|
||||
}
|
||||
switch listed {
|
||||
case "described":
|
||||
if described.reads != 0 {
|
||||
t.Errorf("described: the judging read %d entry objects, want none", described.reads)
|
||||
}
|
||||
case "described-later":
|
||||
if described.meta[assigngraph.EntryObject(running)] != [2]string{"c", "fp-c1"} {
|
||||
t.Errorf("described-later: the entry in use is not put again with its module and fingerprint: %v", described.meta)
|
||||
}
|
||||
reads := described.reads
|
||||
revision(4, shared)
|
||||
k.judge()
|
||||
if described.reads != reads {
|
||||
t.Errorf("described-later: a second judging read %d entry objects again", described.reads-reads)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// What each node runs is heard, never asked (design 47 G §199, §309): the node-engine seat's event `running.<node>.<module>` is
|
||||
// kept last per subject on its own stream, and the assigner's durable consumer on it starts from the last message of every
|
||||
// subject, so a node away still has its last word here. The assigner counts a provider healthy by it (rule 5) and ends a put on it.
|
||||
|
||||
// hearRunning keeps each `running` the stream holds and each it says from now, and wakes the pass on every one.
|
||||
func hearRunning(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
|
||||
acct := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
nh := notHeardWriter(findings, logf)
|
||||
cc, err := acct.HearSeatEvents(ctx, js, nodeengine.StreamRunning, func(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
if err := w.hearRunning(m.Subject(), m.Data()); err != nil {
|
||||
logf("%v", err)
|
||||
return
|
||||
}
|
||||
wake()
|
||||
}, logf)
|
||||
if err != nil {
|
||||
nh.NotHeard(nodeengine.StreamRunning, "what each node runs is not heard until the assigner starts again, so a provider is counted healthy by its have and a put does not end", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.runningHeard = true
|
||||
w.mu.Unlock()
|
||||
nh.Heard(nodeengine.StreamRunning)
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
// hearRunning keeps one `running` event said on subject.
|
||||
func (w *world) hearRunning(subject string, data []byte) error {
|
||||
var r nodeengine.Running
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return fmt.Errorf("%s holds what is not a running: %w", subject, err)
|
||||
}
|
||||
if r.Node == "" || r.Module == "" {
|
||||
return fmt.Errorf("%s names no node or no module, so it is kept for none", subject)
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.running[r.Node+"."+r.Module] = r
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// runningOf is the last `running` heard of module on node.
|
||||
func (w *world) runningOf(node, module string) (nodeengine.Running, bool) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
r, ok := w.running[node+"."+module]
|
||||
return r, ok
|
||||
}
|
||||
|
||||
// runningNow is every `running` heard, by node and module; nil while the seat's events are not heard (the rules then read the
|
||||
// have).
|
||||
func (w *world) runningNow() map[string]map[string]nodeengine.Running {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if !w.runningHeard {
|
||||
return nil
|
||||
}
|
||||
out := map[string]map[string]nodeengine.Running{}
|
||||
for _, r := range w.running {
|
||||
if out[r.Node] == nil {
|
||||
out[r.Node] = map[string]nodeengine.Running{}
|
||||
}
|
||||
out[r.Node][r.Module] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,33 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// The seat mesh-module-assigner's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
|
||||
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
|
||||
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
|
||||
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
|
||||
@@ -47,149 +37,129 @@ func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
}
|
||||
|
||||
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
|
||||
// `<module>.<state>` where the manifest declares them under `reads`.
|
||||
const haveState = "node-engine.have"
|
||||
|
||||
// The indexer's states the verbs read, each asked of the indexer's first-view tool of that name, never read from its bus
|
||||
// state (novox/hq ADR 0596; ADR 0615 §1, ADR 0621 §1, §2: the manifest invokes them, and the indexer answers them to the
|
||||
// assigner alone of the delivery's parts).
|
||||
// `<module>.<state>`.
|
||||
const (
|
||||
indexerModules = "modules"
|
||||
indexerManifests = "manifests"
|
||||
modulesState = delivery.IndexerModule + ".modules"
|
||||
haveState = "node-engine.have"
|
||||
)
|
||||
|
||||
// indexerView is one of the indexer's states whole, asked of its first-view tool through ask (stdio.Ask, or a test's) page
|
||||
// after page, each page continued after the one before's next. It is asked again at every call: the indexer's state is
|
||||
// the truth, and the verbs keep no copy of it.
|
||||
func indexerView(ask func(string, any) (json.RawMessage, error), state string) (map[string]json.RawMessage, error) {
|
||||
tool := buckets.IndexerModule + "." + state
|
||||
if ask == nil {
|
||||
return nil, fmt.Errorf("%s cannot be asked here", tool)
|
||||
}
|
||||
page := views.AskVia(ask, tool)
|
||||
out := map[string]json.RawMessage{}
|
||||
started, pages := time.Now(), 0
|
||||
for after := ""; ; {
|
||||
a, err := page(context.Background(), after)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err)
|
||||
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
|
||||
}
|
||||
pages++
|
||||
maps.Copy(out, a.Values)
|
||||
if a.Next == "" || a.Next == after {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond))
|
||||
return out, nil
|
||||
}
|
||||
after = a.Next
|
||||
}
|
||||
}
|
||||
|
||||
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
|
||||
type seatTools struct {
|
||||
st seatStore
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
now func() time.Time
|
||||
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
|
||||
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
|
||||
seatList *seatlist.Part
|
||||
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
|
||||
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
|
||||
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
|
||||
// direct listing.
|
||||
watched *stdio.WatchedStates
|
||||
}
|
||||
|
||||
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
|
||||
return &seatTools{st: st, ask: ask, now: time.Now}
|
||||
}
|
||||
|
||||
// keyed is one assignments key as a verb answers it.
|
||||
// keyed is one placement key as a verb answers it.
|
||||
type keyed struct {
|
||||
Assignments mma.Assignments `json:"assignments"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Placement delivery.Placement `json:"placement"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
|
||||
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
|
||||
all, err := readAll(s.st, mma.StateAssignments)
|
||||
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
|
||||
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(delivery.PlacementState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("assignments cannot be read: %w", err)
|
||||
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(all))
|
||||
held := mmac.NewKeys()
|
||||
sort.Strings(keys)
|
||||
held := delivery.NewPlacementKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := mmac.ParseKey(key); err != nil {
|
||||
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, all[key].raw, all[key].rev, false); err != nil {
|
||||
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
out, revs := held.AllRevisions()
|
||||
if len(bad) > 0 && len(out) == 0 {
|
||||
return nil, nil, fmt.Errorf("assignments cannot be read: %s", strings.Join(bad, "; "))
|
||||
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
|
||||
}
|
||||
return out, revs, nil
|
||||
}
|
||||
|
||||
// lays is what a proposal is checked against: the modules the indexer holds, as its first view of modules answers them,
|
||||
// and the nodes with a record.
|
||||
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
|
||||
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
mods, err := indexerView(s.ask, indexerModules)
|
||||
mods, err := s.st.keys(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for k := range mods {
|
||||
if name, err := meshcatalogclient.ParseModulesKey(k); err == nil {
|
||||
for _, k := range mods {
|
||||
if name, err := delivery.ParseModulesKey(k); err == nil {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := readAll(s.st, haveState)
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be read: %w", err)
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHaveNode(v.raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
}
|
||||
|
||||
// proposal reads one open proposal.
|
||||
func (s *seatTools) proposal(id string) (mma.Proposal, error) {
|
||||
if err := mmac.CheckProposalID(id); err != nil {
|
||||
return mma.Proposal{}, err
|
||||
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
|
||||
if err := delivery.CheckProposalID(id); err != nil {
|
||||
return delivery.PlacementProposal{}, err
|
||||
}
|
||||
raw, _, found, err := s.st.get(mma.StateProposals, id)
|
||||
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
|
||||
if err != nil {
|
||||
return mma.Proposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
|
||||
}
|
||||
if !found {
|
||||
return mma.Proposal{}, fmt.Errorf("no proposal %s is open: mesh-module-assigner.proposals lists those that are", id)
|
||||
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
|
||||
}
|
||||
return plc.DecodeProposal(raw)
|
||||
return delivery.DecodePlacementProposal(raw)
|
||||
}
|
||||
|
||||
// write writes a proposal laid by assign.Propose, whose write the assigner's process takes as the event to lay it: it writes the
|
||||
// candidates, each with what it adds (design 47 G A-317, A-316); the verb writes none, so no node-engine hears a candidate before
|
||||
// its parts.
|
||||
func (s *seatTools) write(p mma.Proposal) error {
|
||||
p.Written = ptr(s.now().UTC())
|
||||
raw, err := plc.EncodeProposal(p)
|
||||
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
|
||||
// takes as the event to compose and judge it.
|
||||
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
|
||||
p.Written = s.now().UTC()
|
||||
for _, node := range p.Nodes() {
|
||||
key, err := delivery.PlacementCandidateKey(p.ID, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
|
||||
return fmt.Errorf("the candidate %s is not written: %w", key, err)
|
||||
}
|
||||
}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
|
||||
}
|
||||
return nil
|
||||
@@ -197,17 +167,17 @@ func (s *seatTools) write(p mma.Proposal) error {
|
||||
|
||||
// proposed is what propose and adopt answer.
|
||||
type proposed struct {
|
||||
Proposal mma.Proposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Follow string `json:"follow"`
|
||||
}
|
||||
|
||||
func followProposal(id string) string {
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-module-assigner.proposals with id %q says where it stands, and the "+
|
||||
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
|
||||
"operator is asked to approve it once its verdict passes", id)
|
||||
}
|
||||
|
||||
// changesOf reads propose's changes: a JSON array, given as one or as its text.
|
||||
func changesOf(v any) ([]mma.ProposalChange, error) {
|
||||
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
|
||||
var raw []byte
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
@@ -220,7 +190,7 @@ func changesOf(v any) ([]mma.ProposalChange, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out []mma.ProposalChange
|
||||
var out []delivery.PlacementProposalChange
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
@@ -236,92 +206,32 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == mmac.OpSeed {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
started := time.Now()
|
||||
step := func(what string, err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s failed after %s: %v\n", by, what, time.Since(started).Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s at %s\n", by, what, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
current, revs, err := s.assignments()
|
||||
step("assignments read", err)
|
||||
current, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
step("the indexer's modules and the nodes' records read", err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warrant, err := warrantOf(args["warrant"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
enrollers := plc.EnrollersOf(current)
|
||||
now := s.now()
|
||||
p, _, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant, SeatListRead: s.seatList.Heard()})
|
||||
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
step(fmt.Sprintf("refused (%d findings)", len(findings)), nil)
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
step("judged", nil)
|
||||
if err := s.write(p); err != nil {
|
||||
step("the proposal "+p.ID+" written", err)
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
step("the proposal "+p.ID+" written", nil)
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
|
||||
func warrantOf(v any) (*mma.Warrant, error) {
|
||||
var raw []byte
|
||||
switch w := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
if strings.TrimSpace(w) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw = []byte(w)
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out mma.Warrant
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: the holder of the seat
|
||||
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
|
||||
// `person.<module>`), else the caller as proven, which the refusal names.
|
||||
func moduleOfCaller(by string, enrollers []string) string {
|
||||
// The enrolment's caller stays (design 47 H-A-41): a module's bus user is `<node>.<module>`, a person's `person.<name>`.
|
||||
_, module, ok := strings.Cut(by, ".")
|
||||
if ok && !strings.HasPrefix(by, "person.") && slices.Contains(enrollers, module) {
|
||||
return module
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// assignments-withdrawn carrying it whole.
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
@@ -332,11 +242,11 @@ func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
|
||||
raw, err := plc.EncodeProposal(p)
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
|
||||
@@ -349,46 +259,107 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Again = ptr(s.now().UTC())
|
||||
raw, err := plc.EncodeProposal(p)
|
||||
p.Again = s.now().UTC()
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(mma.StateProposals, p.ID, raw); err != nil {
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
|
||||
}
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// doPlacement is the verb placement (design 47 G A-62, §704): the module's placements as the assignments rows hold them, one
|
||||
// per branch a row names, with the nodes that run the module from it. A row naming no branch is one placement with the branch
|
||||
// empty, as the row holds it.
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
module := arg(args, "module")
|
||||
if module == "" {
|
||||
return nil, errors.New("placement takes the module")
|
||||
}
|
||||
all, _, err := s.assignments()
|
||||
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
|
||||
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
|
||||
|
||||
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
|
||||
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
current, _, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byBranch := map[string][]string{}
|
||||
for node, a := range all {
|
||||
for _, m := range a.Modules {
|
||||
if m.Module == module {
|
||||
byBranch[m.Branch] = append(byBranch[m.Branch], node)
|
||||
if len(current) > 0 {
|
||||
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
|
||||
}
|
||||
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
|
||||
for _, id := range ids {
|
||||
if p, err := s.proposal(id); err == nil && p.Seed {
|
||||
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
out := mma.PlacementAnswer{Placements: []mma.ModulePlacement{}}
|
||||
for _, branch := range slices.Sorted(maps.Keys(byBranch)) {
|
||||
out.Placements = append(out.Placements, mma.ModulePlacement{Branch: branch, Nodes: slices.Sorted(slices.Values(byBranch[branch]))})
|
||||
raw, err := s.ask(exportCall, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
export, err := assign.ExportOf(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// assignmentsAnswer is what the verb assignments answers.
|
||||
type assignmentsAnswer struct {
|
||||
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
|
||||
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
|
||||
// unreadable, is not handed away.
|
||||
func (s *seatTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
if s.leased() {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("hand-back says why: it is recorded with the hand act")
|
||||
}
|
||||
if by == "" {
|
||||
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
|
||||
}
|
||||
_, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return nil, errors.New("there is no placement key to hand back")
|
||||
}
|
||||
now := s.now()
|
||||
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
|
||||
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the hand-back is not written: %w", err)
|
||||
}
|
||||
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
|
||||
"node returns to the controller's compositions"}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
Count int `json:"count"`
|
||||
@@ -399,38 +370,38 @@ type assignmentsAnswer struct {
|
||||
|
||||
// historyEntry is one change of a key in the state's revisions.
|
||||
type historyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change plc.Change `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Revision uint64 `json:"revision"`
|
||||
At time.Time `json:"at"`
|
||||
Change delivery.PlacementChange `json:"change"`
|
||||
Deleted bool `json:"deleted,omitempty"`
|
||||
}
|
||||
|
||||
// doAssignments is the verb assignments (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
|
||||
// changes in order from the revisions the state keeps.
|
||||
func (s *seatTools) doAssignments(args map[string]any) (any, error) {
|
||||
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
|
||||
if arg(args, "seats") == "true" {
|
||||
return s.seats(arg(args, "node"))
|
||||
}
|
||||
all, revs, err := s.assignments()
|
||||
all, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, module := arg(args, "node"), arg(args, "module")
|
||||
out := assignmentsAnswer{Keys: map[string]keyed{}, Count: len(all), History: plc.AssignmentsHistory}
|
||||
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
|
||||
for n, p := range all {
|
||||
if node != "" && n != node {
|
||||
continue
|
||||
}
|
||||
if module != "" && !strings.Contains(" "+strings.Join(mmac.Assigned(p), " ")+" ", " "+module+" ") {
|
||||
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
|
||||
continue
|
||||
}
|
||||
out.Keys[n] = keyed{Assignments: p, Revision: revs[n]}
|
||||
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
|
||||
out.Nodes = append(out.Nodes, n)
|
||||
}
|
||||
sort.Strings(out.Nodes)
|
||||
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
|
||||
return nil, fmt.Errorf("%s has no assignments key", node)
|
||||
return nil, fmt.Errorf("%s has no placement key", node)
|
||||
}
|
||||
if arg(args, "history") != "true" {
|
||||
return out, nil
|
||||
@@ -468,8 +439,8 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
|
||||
out.Changes = append(out.Changes, e)
|
||||
}
|
||||
}
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its assignments-changed event on "+
|
||||
"the seat's subject on the shared events stream, kept %d days", plc.AssignmentsHistory, plc.EventDays)
|
||||
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
|
||||
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -477,38 +448,42 @@ func (s *seatTools) doAssignments(args map[string]any) (any, error) {
|
||||
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
|
||||
// every node's answer now.
|
||||
type openProposal struct {
|
||||
Proposal mma.Proposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
|
||||
Proposal delivery.PlacementProposal `json:"proposal"`
|
||||
Candidates map[string]uint64 `json:"candidates"`
|
||||
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
|
||||
}
|
||||
|
||||
// doProposals is the verb proposals (§10).
|
||||
func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
ids, err := s.st.keys(mma.StateProposals)
|
||||
ids, err := s.st.keys(delivery.PlacementProposalsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
|
||||
}
|
||||
cands, err := s.st.keys(mma.StateCandidates)
|
||||
cands, err := s.st.keys(delivery.PlacementCandidatesState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := readAll(s.st, haveState)
|
||||
haves, err := s.st.keys(haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be read: %w", err)
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
h, err := nodeengineclient.DecodeHaveNode(v.raw)
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for id, a := range h.Candidates {
|
||||
if answers[id] == nil {
|
||||
answers[id] = map[string]nodeengine.CandidateAnswer{}
|
||||
answers[id] = map[string]delivery.CandidateAnswer{}
|
||||
}
|
||||
answers[id][node] = a
|
||||
}
|
||||
@@ -525,11 +500,11 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
}
|
||||
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
|
||||
for _, k := range cands {
|
||||
pid, node, err := plc.ParseCandidateKey(k)
|
||||
pid, node, err := delivery.ParsePlacementCandidateKey(k)
|
||||
if err != nil || pid != id {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := s.st.get(mma.StateCandidates, k); err == nil && found {
|
||||
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
|
||||
o.Candidates[node] = rev
|
||||
}
|
||||
}
|
||||
@@ -544,61 +519,50 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
|
||||
// only from a tool of that name), with the seat's own schemas.
|
||||
func (s *seatTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
|
||||
input := map[string]map[string]any{}
|
||||
description := map[string]string{}
|
||||
for _, v := range seat.Serves {
|
||||
input[v.Name], description[v.Name] = v.Input, v.Description
|
||||
}
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
mma.VerbAssignments: s.doAssignments,
|
||||
mma.VerbProposals: s.doProposals,
|
||||
mma.VerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
mma.VerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
mma.VerbAgain: s.doAgain,
|
||||
mma.VerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbPlacement: s.doPlacement,
|
||||
catalogue.PlacementVerbProposals: s.doProposals,
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range mma.Verbs() {
|
||||
out = append(out, stdio.Tool{Name: mma.Seat + "." + v, Run: run[v]})
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
|
||||
// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
|
||||
func (s *seatTools) seatHolders() (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
|
||||
return vals, nil
|
||||
}
|
||||
}
|
||||
keys, err := s.st.keys(mma.StateSeatHolders)
|
||||
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
keys, err := s.st.keys(delivery.SeatHoldersState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
|
||||
}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
sort.Strings(keys)
|
||||
out := []delivery.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
|
||||
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
|
||||
}
|
||||
if found {
|
||||
out[key] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
|
||||
// holds it; with node, the keys of seats held on that node.
|
||||
func (s *seatTools) seats(node string) (any, error) {
|
||||
raws, err := s.seatHolders()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(raws))
|
||||
out := []mma.SeatHolders{}
|
||||
for _, key := range keys {
|
||||
var sh mma.SeatHolders
|
||||
if json.Unmarshal(raws[key], &sh) != nil {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
|
||||
var sh delivery.SeatHolders
|
||||
if json.Unmarshal(raw, &sh) != nil {
|
||||
continue
|
||||
}
|
||||
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
|
||||
continue
|
||||
}
|
||||
out = append(out, sh)
|
||||
@@ -606,56 +570,76 @@ func (s *seatTools) seats(node string) (any, error) {
|
||||
return map[string]any{"seats": out, "holds": len(keys)}, nil
|
||||
}
|
||||
|
||||
// keyedRevision is one revision of a node's assignments key, the key it is of beside it.
|
||||
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
|
||||
type keyedRevision struct {
|
||||
stdio.StateRevision
|
||||
Key string
|
||||
}
|
||||
|
||||
// nodeHistory is the revisions of a node's assignments, in the order the bus wrote them: its node key's `<node>.node`,
|
||||
// whose every revision is one write of the node (research 072 row 37: the node key is a write's commit, so its history is
|
||||
// the node's assignments history).
|
||||
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
|
||||
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
|
||||
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
|
||||
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
|
||||
nodeKey, err := mmac.NodeKey(node)
|
||||
legacy, err := delivery.LegacyPlacementKey(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
revisions, err := s.st.history(mma.StateAssignments, nodeKey)
|
||||
nodeKey, err := delivery.PlacementNodeKey(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", nodeKey, err)
|
||||
return nil, err
|
||||
}
|
||||
var out []keyedRevision
|
||||
for _, r := range revisions {
|
||||
out = append(out, keyedRevision{StateRevision: r, Key: nodeKey})
|
||||
for _, key := range []string{legacy, nodeKey} {
|
||||
revisions, err := s.st.history(delivery.PlacementState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, r := range revisions {
|
||||
out = append(out, keyedRevision{StateRevision: r, Key: key})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sketch is a revision of a node's assignments as far as its history says a change: each module's digest, and its pins
|
||||
// and holdings, which change every module (plc.ChangeOf).
|
||||
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
|
||||
// and holdings, which change every module (delivery.ChangeOf).
|
||||
type sketch struct {
|
||||
modules []mma.AssignmentsByNodeNodeValueModulesItem
|
||||
modules []delivery.PlacementRef
|
||||
nodeWide string
|
||||
}
|
||||
|
||||
// sketchOf reads one revision of a node key; a value that does not read places nothing.
|
||||
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
|
||||
func sketchOf(key string, value []byte) sketch {
|
||||
_, _, kind, err := mmac.ParseKey(key)
|
||||
if err != nil || kind != mmac.NodeKind {
|
||||
return sketch{}
|
||||
}
|
||||
n, err := mmac.DecodeNode(value)
|
||||
_, _, kind, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
raw, _ := json.Marshal([]any{n.Pins, n.Holds})
|
||||
return sketch{modules: n.Modules, nodeWide: string(raw)}
|
||||
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
|
||||
raw, _ := json.Marshal([]any{pins, holds})
|
||||
return string(raw)
|
||||
}
|
||||
if kind == delivery.PlacementNodeKind {
|
||||
n, err := delivery.DecodePlacementNode(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
return sketch{}
|
||||
}
|
||||
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
|
||||
for _, m := range p.Modules {
|
||||
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changeFrom is what changed from before to s on node, as plc.ChangeOf says it.
|
||||
func (s sketch) changeFrom(node string, before sketch) plc.Change {
|
||||
c := plc.Change{Node: node}
|
||||
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
|
||||
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
|
||||
c := delivery.PlacementChange{Node: node}
|
||||
was := map[string]string{}
|
||||
for _, r := range before.modules {
|
||||
was[r.Module] = r.Digest
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +0,0 @@
|
||||
package main
|
||||
|
||||
// The assigner reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
|
||||
// reader over the seat mesh-catalog's verb seat-definitions and its two events, its tools `assigner_seat_list` and
|
||||
// `assigner_seat_list_again`. What the reader holds is given to catalogue at every change, which the check of a proposal and the
|
||||
// seat-holders state read; until the list is read, a mesh seat the bootstrapped definitions do not name is never judged held once
|
||||
// (plc.CheckAssignments), and seat-holders is not written.
|
||||
@@ -1,114 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||
)
|
||||
|
||||
// testDefinitions are seatlisttest's, parsed once.
|
||||
var testDefinitions = sync.OnceValue(func() []seatdef.Definition { return seatlisttest.Definitions() })
|
||||
|
||||
// standInCatalog is the seat mesh-catalog as a test's reader asks it (ADR 0636 §4): every definition of seatlisttest, or,
|
||||
// silent, no answer at all.
|
||||
type standInCatalog struct{ silent bool }
|
||||
|
||||
func (c standInCatalog) Ask(key string, body any) (json.RawMessage, error) {
|
||||
if c.silent || key != seatlist.Key() {
|
||||
return nil, errors.New("no responders")
|
||||
}
|
||||
defs := testDefinitions()
|
||||
seat, _ := body.(map[string]any)["seat"].(string)
|
||||
if seat == "" {
|
||||
l := seatlist.Listing{Revision: 1}
|
||||
for _, d := range defs {
|
||||
l.Seats = append(l.Seats, seatlist.Entry{Seat: d.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + d.Seat})
|
||||
}
|
||||
return json.Marshal(l)
|
||||
}
|
||||
for _, d := range defs {
|
||||
if d.Seat == seat {
|
||||
raw, err := json.Marshal(d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(seatlist.Answer{Seat: seat, Found: true, Revision: 1,
|
||||
Entry: &seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "d-" + seat, Definition: raw}})
|
||||
}
|
||||
}
|
||||
return json.Marshal(seatlist.Answer{Seat: seat, Revision: 1})
|
||||
}
|
||||
|
||||
func (standInCatalog) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
|
||||
|
||||
// gateOver is a seat gate following the stand-in, once its first read has ended. A read one gives catalogue the test list
|
||||
// (catalogue.UseSeats), as a host's does, for the tests after it too: the list is the same everywhere.
|
||||
func gateOver(t *testing.T, silent bool) *seatlist.Part {
|
||||
t.Helper()
|
||||
if silent {
|
||||
// A silent seat reached no one: its reader asks again within the bus's retry bound (issue 1966), which a test
|
||||
// that waits for it to stay unread keeps short.
|
||||
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 5*time.Millisecond))
|
||||
}
|
||||
g := seatlist.NewPart("assigner")
|
||||
r := g.Follow(standInCatalog{silent: silent}, nil, t.Logf)
|
||||
if r == nil {
|
||||
t.Fatal("the reader did not start")
|
||||
}
|
||||
<-r.Ready()
|
||||
if g.Heard() == silent {
|
||||
t.Fatalf("the gate read %v over a stand-in silent %v: %s", g.Heard(), silent, g.Health())
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Design 47 G §204: the seat list's reader is wired once, by mesh-sdk's go/seatlist.Part; no part wires its own. A search of the
|
||||
// assigner's code finds no start of a reader and no tools of its own.
|
||||
func TestTheSeatListIsWiredByTheSDKAlone(t *testing.T) {
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, own := range []string{"seatlist.Start(", "seatlist.Tools(", "OnChange:"} {
|
||||
if strings.Contains(string(raw), own) {
|
||||
t.Errorf("%s wires the seat list itself (%s), where mesh-sdk's seatlist.Part does it", f, own)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A-179, A-186: the definition the verb input's callers are checked against is the seat list's, none before the reader starts
|
||||
// or while the seat mesh-catalog has not answered, and none for a seat the list does not hold.
|
||||
func TestTheAnsweredSeatsDefinitionIsTheSeatListsOnceRead(t *testing.T) {
|
||||
if _, ok := seatlist.Definition(seatlist.NewPart("assigner").Reader, "mesh-wants")(); ok {
|
||||
t.Error("a definition is answered before the seat list's reader started")
|
||||
}
|
||||
if _, ok := seatlist.Definition(gateOver(t, true).Reader, "mesh-wants")(); ok {
|
||||
t.Error("a definition is answered while the seat mesh-catalog has not answered the list")
|
||||
}
|
||||
read := gateOver(t, false)
|
||||
if d, ok := seatlist.Definition(read.Reader, "mesh-wants")(); !ok || d.Seat != "mesh-wants" {
|
||||
t.Errorf("the read list's mesh-wants is answered as %v, %v", d.Seat, ok)
|
||||
}
|
||||
if _, ok := seatlist.Definition(read.Reader, "no-such-seat")(); ok {
|
||||
t.Error("a seat the list does not hold is answered a definition")
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module a candidate adds with no release heard is said once, not each pass.
|
||||
func TestASkippedAddIsSaidOncePerModuleAndFingerprint(t *testing.T) {
|
||||
var logs []string
|
||||
p := &passer{logf: func(f string, a ...any) { logs = append(logs, strings.TrimSpace(f)) }}
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp2", "why")
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("said %d times, want 2: %v", len(logs), logs)
|
||||
}
|
||||
}
|
||||
@@ -1,318 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2,
|
||||
// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's
|
||||
// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key
|
||||
// <seat>.<ask>, for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an
|
||||
// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask
|
||||
// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped.<ask> for
|
||||
// ask-taken-bound, and either ends its step could not run naming the drop and who asked it.
|
||||
|
||||
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
|
||||
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
|
||||
const (
|
||||
askTakenBoundSetting = "ask-taken-bound"
|
||||
defaultAskTakenBound = 10 * time.Minute
|
||||
)
|
||||
|
||||
// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404).
|
||||
const droppedPrefix = "dropped."
|
||||
|
||||
// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet.
|
||||
type takenStore interface {
|
||||
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
RenewTTL(key string, value any, ttl time.Duration) (uint64, error)
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg).
|
||||
type queueMsg interface {
|
||||
Data() []byte
|
||||
Headers() nats.Header
|
||||
Ack() error
|
||||
Nak() error
|
||||
Term() error
|
||||
InProgress() error
|
||||
}
|
||||
|
||||
// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end.
|
||||
type askDrop struct {
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
func (d *askDrop) Error() string {
|
||||
by := d.By
|
||||
if by == "" {
|
||||
by = "its asker"
|
||||
}
|
||||
return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why)
|
||||
}
|
||||
|
||||
// askIDOf is the ask's id a put carries: its event id, else its message id.
|
||||
func askIDOf(h nats.Header) string {
|
||||
if id := h.Get("x-event-id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return h.Get(nats.MsgIdHdr)
|
||||
}
|
||||
|
||||
// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is
|
||||
// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed
|
||||
// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken
|
||||
// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run
|
||||
// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice.
|
||||
func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) {
|
||||
id := askIDOf(m.Headers())
|
||||
var run meshwants.StepRun
|
||||
err := json.Unmarshal(m.Data(), &run)
|
||||
how := c.howOf(run.Step, false)
|
||||
if err != nil || id == "" || how == nil {
|
||||
c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if d := c.droppedBefore(id); d != nil {
|
||||
c.hold(id, m)
|
||||
c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d)
|
||||
c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "")
|
||||
if err := c.taken.Delete(droppedPrefix + id); err != nil {
|
||||
c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if c.running(id) {
|
||||
c.hold(id, m)
|
||||
c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
|
||||
return
|
||||
}
|
||||
first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound)
|
||||
if err != nil {
|
||||
c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
c.hold(id, m)
|
||||
if first {
|
||||
at := c.now().UTC()
|
||||
if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil {
|
||||
c.logf("saying ask-taken of %s: %v", id, err)
|
||||
}
|
||||
} else {
|
||||
c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline)
|
||||
}
|
||||
go c.say1(ctx, id, run, how, "")
|
||||
}
|
||||
|
||||
// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs.
|
||||
func (c *compositionSteps) hold(id string, m queueMsg) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.held == nil {
|
||||
c.held = map[string][]queueMsg{}
|
||||
}
|
||||
c.held[id] = append(c.held[id], m)
|
||||
}
|
||||
|
||||
// settle acknowledges (ack) or hands again (nak) every message held for id.
|
||||
func (c *compositionSteps) settle(id string, ack bool) {
|
||||
c.mu.Lock()
|
||||
ms := c.held[id]
|
||||
delete(c.held, id)
|
||||
c.mu.Unlock()
|
||||
for _, m := range ms {
|
||||
if ack {
|
||||
_ = m.Ack()
|
||||
} else {
|
||||
_ = m.Nak()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inProgressEvery says every message held for id in progress every d, until the returned stop is called.
|
||||
func (c *compositionSteps) inProgressEvery(id string) (stop func()) {
|
||||
every := time.Duration(0)
|
||||
if c.inProgress != nil {
|
||||
every = c.inProgress()
|
||||
}
|
||||
if every <= 0 {
|
||||
return func() {}
|
||||
}
|
||||
ticks := c.ticks
|
||||
if ticks == nil {
|
||||
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
|
||||
}
|
||||
tick, stopTicks := ticks(every)
|
||||
done, ended := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(ended)
|
||||
defer stopTicks()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick:
|
||||
c.mu.Lock()
|
||||
ms := append([]queueMsg(nil), c.held[id]...)
|
||||
c.mu.Unlock()
|
||||
for _, m := range ms {
|
||||
_ = m.InProgress()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { close(done); <-ended }
|
||||
}
|
||||
|
||||
// droppedBefore is the drop kept for an ask not pulled yet, nil for none.
|
||||
func (c *compositionSteps) droppedBefore(id string) *askDrop {
|
||||
if c.taken == nil {
|
||||
return nil
|
||||
}
|
||||
e, err := c.taken.Get(droppedPrefix + id)
|
||||
if err != nil {
|
||||
c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err)
|
||||
return nil
|
||||
}
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var d askDrop
|
||||
if json.Unmarshal(e.Value, &d) != nil {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
|
||||
// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and
|
||||
// ends could not run naming it; one not running here is kept in taken as dropped.<ask> for ask-taken-bound, so its pull ends
|
||||
// it the same way. It answers whether the ask was held here.
|
||||
func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) {
|
||||
id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why)
|
||||
if id == "" || why == "" {
|
||||
return nil, errors.New("give the ask's id and why it is dropped")
|
||||
}
|
||||
by := strings.TrimSpace(in.By)
|
||||
if by == "" {
|
||||
by = caller
|
||||
}
|
||||
d := &askDrop{By: by, Why: why, At: c.now().UTC()}
|
||||
c.mu.Lock()
|
||||
stop := c.stops[id]
|
||||
c.mu.Unlock()
|
||||
if stop != nil {
|
||||
stop(d)
|
||||
c.logf("the ask %s running here is dropped: %v", id, d)
|
||||
return meshwants.DropAnswer{"held": true}, nil
|
||||
}
|
||||
if c.taken == nil {
|
||||
return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id)
|
||||
}
|
||||
if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil {
|
||||
return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err)
|
||||
}
|
||||
c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d)
|
||||
return meshwants.DropAnswer{"held": false}, nil
|
||||
}
|
||||
|
||||
// dropVerb is the process's handler of the seat mesh-wants' act drop.
|
||||
func dropVerb(c *compositionSteps) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.DropInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return c.dropAsk(caller, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
|
||||
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401).
|
||||
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
|
||||
return func() time.Duration {
|
||||
if d, ok := def(); ok {
|
||||
if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
|
||||
return time.Duration(worker.InProgressSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return ackWait() / 3
|
||||
}
|
||||
}
|
||||
|
||||
// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
|
||||
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
|
||||
func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error {
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)
|
||||
var once sync.Once
|
||||
bind := func() bool {
|
||||
cons, err := js.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
bound := false
|
||||
once.Do(func() {
|
||||
bound = true
|
||||
info := cons.CachedInfo()
|
||||
if info != nil && info.Config.AckWait > 0 {
|
||||
ackWait(info.Config.AckWait)
|
||||
}
|
||||
pending := 1000
|
||||
if info != nil && info.Config.MaxAckPending > 0 {
|
||||
pending = info.Config.MaxAckPending
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending))
|
||||
if err != nil {
|
||||
c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err)
|
||||
return
|
||||
}
|
||||
c.logf("the composition step is pulled from %s on %s", worker, stream)
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
})
|
||||
return bound
|
||||
}
|
||||
if bind() {
|
||||
return nil
|
||||
}
|
||||
c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream)
|
||||
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err)
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
|
||||
if bind() { // made between the first look and the subscription
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askTakenBound reads the setting ask-taken-bound (§781).
|
||||
func askTakenBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the assigner's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// kept is one key's value and revision, as a whole state read gives it.
|
||||
type kept struct {
|
||||
raw []byte
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// pager is a store that reads a state's values many to a request (stdio.KeptState.Page): the runtime's store.
|
||||
type pager interface {
|
||||
page(state, after string) (stdio.StatePage, error)
|
||||
}
|
||||
|
||||
// pageBytes and pageBudget bound one page of a whole state read, so its answer fits one bus message and its time.
|
||||
const (
|
||||
pageBytes = 768 * 1024
|
||||
pageBudget = 10 * time.Second
|
||||
)
|
||||
|
||||
func (runtimeStore) page(state, after string) (stdio.StatePage, error) {
|
||||
return stdio.State(state).Page(after, pageBytes, pageBudget)
|
||||
}
|
||||
|
||||
// readAll is every key of a state with its value and revision: in pages when the store has them, else key by key. Key by
|
||||
// key cost one bus request each, so the assignments (about 330 keys) took 27 s to read and a proposal outlasted its
|
||||
// caller's 30 s (2026-11-14), as the indexer's view of the trunk had the hour before.
|
||||
func readAll(st store, state string) (map[string]kept, error) {
|
||||
out := map[string]kept{}
|
||||
if p, ok := st.(pager); ok {
|
||||
for after := ""; ; {
|
||||
pg, err := p.page(state, after)
|
||||
if errors.Is(err, errors.ErrUnsupported) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", state, err)
|
||||
}
|
||||
for _, e := range pg.Entries {
|
||||
out[e.Key] = kept{raw: e.Value, rev: e.Revision}
|
||||
}
|
||||
if pg.Next == "" || pg.Next == after {
|
||||
return out, nil
|
||||
}
|
||||
after = pg.Next
|
||||
}
|
||||
}
|
||||
keys, err := st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
raw, rev, found, err := st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = kept{raw: raw, rev: rev}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,368 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// One pipeline's wants are written tier by tier (design 47 H §751, H-A-10, H-A-56): on a passed pipeline-ended the assigner
|
||||
// writes the wants of its first tier's movers; once each of them runs at the mover's fingerprint (the node-engine seat's
|
||||
// `running`, H-A-40) on every present placing machine, it writes the next tier's, and so on. The tier progress is the key
|
||||
// `pipeline.<branch>.<source>` of the state written, `{tier, since, movers}`; a tier counts placed movers only, and a machine away
|
||||
// is not waited for. A tier whose movers do not all run within `tier-bound` ends the delivery loudly: no later tier is written,
|
||||
// the finding `tier-late` (scope delivery, id the pipeline) is raised naming the movers and machines not running, and
|
||||
// `pipeline-wanted` says outcome tier-late; once every tier runs it says outcome running and the key goes (H-A-75). tier-late
|
||||
// clears when the late movers run or a later passed pipeline moves them (H-A-56).
|
||||
|
||||
// tierPrefix starts the key of a pipeline's tier progress in the state written.
|
||||
const tierPrefix = "pipeline."
|
||||
|
||||
// kindTierLate is the finding of a tier not running within tier-bound (design 47 H §751).
|
||||
const kindTierLate = "tier-late"
|
||||
|
||||
// defaultTierBound is tier-bound's default, 30 minutes (design 47 H §751).
|
||||
const defaultTierBound = 30 * time.Minute
|
||||
|
||||
// tierMover is one mover of a passed pipeline whose version the state written keeps: its module, fingerprint and tier.
|
||||
type tierMover struct {
|
||||
Module string `json:"module"`
|
||||
Fp string `json:"fp"`
|
||||
Tier int64 `json:"tier"`
|
||||
// NoVersion is a passed mover whose version is not yet written: its tier does not run until it is (H-A-277 (1)).
|
||||
NoVersion bool `json:"no-version,omitempty"`
|
||||
}
|
||||
|
||||
// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source
|
||||
// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a
|
||||
// value kept before the field, read as the key's number.
|
||||
type tierProgress struct {
|
||||
Tier int64 `json:"tier"`
|
||||
Since time.Time `json:"since"`
|
||||
Movers []tierMover `json:"movers"`
|
||||
Source int64 `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
|
||||
func tierKey(pipeline string) string { return tierPrefix + pipeline }
|
||||
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form
|
||||
// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's
|
||||
// is (a module's end, `<module>.<branch>`, is none).
|
||||
func isTierKey(key string) bool {
|
||||
rest, ok := strings.CutPrefix(key, tierPrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := pipelinekey.Parse(rest)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
|
||||
// lowest of theirs, since now (H-A-277 (1)). A pipeline heard again adds to the progress that stands each mover it does not hold,
|
||||
// at the mover's tier, and marks versioned each held mover whose version is now written (H-A-270, H-A-273): its want is written
|
||||
// when the progress reaches its tier, or at once when that tier ran already, never out of tier order.
|
||||
func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover, versioned map[string]bool) error {
|
||||
for {
|
||||
t, rev, ok := e.tiersOf(pipeline)
|
||||
if !ok {
|
||||
if len(movers) == 0 {
|
||||
return nil
|
||||
}
|
||||
first := movers[0].Tier
|
||||
for _, m := range movers {
|
||||
first = min(first, m.Tier)
|
||||
}
|
||||
t = tierProgress{Tier: first, Since: e.clock().UTC(), Source: source}
|
||||
}
|
||||
moved := false
|
||||
for i := range t.Movers {
|
||||
if t.Movers[i].NoVersion && versioned[t.Movers[i].Module] {
|
||||
t.Movers[i].NoVersion, moved = false, true
|
||||
}
|
||||
}
|
||||
for _, m := range movers {
|
||||
if !slices.ContainsFunc(t.Movers, func(h tierMover) bool { return h.Module == m.Module }) {
|
||||
t.Movers, moved = append(t.Movers, m), true
|
||||
}
|
||||
}
|
||||
if !moved {
|
||||
return nil
|
||||
}
|
||||
value, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = e.st.swap(tierKey(pipeline), value, rev)
|
||||
if errors.Is(err, errSwapped) {
|
||||
if _, _, held := e.tiersOf(pipeline); !ok && !held {
|
||||
return nil // a key that stands but does not read is left as it is
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == nil && ok {
|
||||
e.logf("pipeline %s: a mover whose version came after its end is taken into the tier progress", pipeline)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (e *pipelineEnds) clock() time.Time {
|
||||
if e.now != nil {
|
||||
return e.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// tiersOf is the tier progress of pipeline, with its revision; false when none stands.
|
||||
func (e *pipelineEnds) tiersOf(pipeline string) (tierProgress, uint64, bool) {
|
||||
raw, rev, err := e.st.get(tierKey(pipeline))
|
||||
var t tierProgress
|
||||
if err != nil || raw == nil || json.Unmarshal(raw, &t) != nil {
|
||||
return tierProgress{}, 0, false
|
||||
}
|
||||
return t, rev, true
|
||||
}
|
||||
|
||||
// tierWritten says whether the want of an end may be written: its pipeline's tier progress reached the end's rank, or no
|
||||
// progress stands for its pipeline (every tier ran).
|
||||
func (e *pipelineEnds) tierWritten(end End) bool {
|
||||
t, _, ok := e.tiersOf(end.Pipeline)
|
||||
return !ok || end.Rank <= t.Tier
|
||||
}
|
||||
|
||||
// tierBound is the setting tier-bound; zero or less is its default.
|
||||
func (p *passer) tierBoundOf() time.Duration {
|
||||
if p.tierBound > 0 {
|
||||
return p.tierBound
|
||||
}
|
||||
return defaultTierBound
|
||||
}
|
||||
|
||||
// tiers moves each pipeline's tier progress (design 47 H §751): a tier whose placed movers all run on every present placing machine
|
||||
// gives way to the next, the last to the end said `running`; a tier past tier-bound is late. placedOn is, by module, the nodes
|
||||
// placing it on branch. It returns the earliest deadline still to come (zero: none), for one timer to wake the pass at.
|
||||
func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []string) time.Time {
|
||||
if p.ends == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
ks, err := p.ends.st.keys()
|
||||
if err != nil {
|
||||
p.logf("the tier progress cannot be listed: %v", err)
|
||||
return time.Time{}
|
||||
}
|
||||
sort.Strings(ks)
|
||||
now := p.clock().UTC()
|
||||
bound := p.tierBoundOf()
|
||||
var next time.Time
|
||||
for _, key := range ks {
|
||||
if !isTierKey(key) {
|
||||
continue
|
||||
}
|
||||
pipeline := strings.TrimPrefix(key, tierPrefix)
|
||||
pk, err := pipelinekey.Parse(pipeline)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10)
|
||||
for {
|
||||
t, rev, ok := p.ends.tiersOf(pipeline)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier)
|
||||
late := !now.Before(t.Since.Add(bound))
|
||||
if late {
|
||||
p.tierLate(pipeline, branch, number, t, notRunning)
|
||||
break
|
||||
}
|
||||
if len(notRunning) > 0 {
|
||||
if d := t.Since.Add(bound); next.IsZero() || d.Before(next) {
|
||||
next = d
|
||||
}
|
||||
break
|
||||
}
|
||||
// The tier runs: the next tier's wants are written, or every tier ran.
|
||||
later := int64(0)
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier > t.Tier && (later == 0 || m.Tier < later) {
|
||||
later = m.Tier
|
||||
}
|
||||
}
|
||||
if later == 0 {
|
||||
if err := p.ends.st.drop(key); err != nil {
|
||||
p.logf("the tier progress %s is not deleted: %v", key, err)
|
||||
break
|
||||
}
|
||||
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
break
|
||||
}
|
||||
value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source})
|
||||
if err == nil {
|
||||
err = p.ends.st.swap(key, value, rev)
|
||||
}
|
||||
if errors.Is(err, errSwapped) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the tier progress %s is not moved to tier %d: %v", key, later, err)
|
||||
break
|
||||
}
|
||||
p.logf("pipeline %s: tier %d runs on every present placing machine, so tier %d's wants are written", pipeline, t.Tier, later)
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// notRunning is, of tier's movers in t whose kept end still names pipeline, those not running at their fingerprint on a present
|
||||
// placing machine, with those machines. A mover a later pipeline moved counts as run; one placed nowhere counts as run.
|
||||
func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) []string, pipeline, branch string, t tierProgress, tier int64) []meshwants.NotRunning {
|
||||
var out []meshwants.NotRunning
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier != tier {
|
||||
continue
|
||||
}
|
||||
if m.NoVersion {
|
||||
// A mover whose version is not yet written holds its tier, under the tier bound (H-A-277 (1)).
|
||||
machines := []string{}
|
||||
for _, node := range placedOn(m.Module, branch) {
|
||||
if online, known := n.Online[node]; !known || online {
|
||||
machines = append(machines, node)
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
|
||||
continue
|
||||
}
|
||||
if end, ok := p.ends.endOf(m.Module, branch); !ok || end.Pipeline != pipeline {
|
||||
continue // a later pipeline moved it
|
||||
}
|
||||
var machines []string
|
||||
for _, node := range placedOn(m.Module, branch) {
|
||||
if online, known := n.Online[node]; known && !online {
|
||||
continue // a machine away is not waited for
|
||||
}
|
||||
r, ok := n.Running[node][m.Module]
|
||||
if !ok || r.Stopped || r.Fingerprint != m.Fp {
|
||||
machines = append(machines, node)
|
||||
}
|
||||
}
|
||||
if len(machines) > 0 {
|
||||
sort.Strings(machines)
|
||||
out = append(out, meshwants.NotRunning{Module: m.Module, Machines: machines})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late,
|
||||
// once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is
|
||||
// written, until no later tier's mover is kept at this pipeline any more.
|
||||
func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) {
|
||||
o := tierLateFinding(pipeline, t.Tier, notRunning)
|
||||
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
|
||||
if len(notRunning) > 0 {
|
||||
if !p.said[key+".raised"] {
|
||||
if p.findings.Put != nil {
|
||||
if err := p.findings.Put(o.Finding(p.clock())); err != nil {
|
||||
p.logf("the finding %s is not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.said[key+".raised"] = true
|
||||
delete(p.said, key+".cleared")
|
||||
p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary)
|
||||
p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning)
|
||||
}
|
||||
} else if !p.said[key+".cleared"] {
|
||||
if p.findings.Delete != nil {
|
||||
if err := p.findings.Delete(key); err != nil {
|
||||
p.logf("the finding %s is not deleted: %v", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.said[key+".cleared"] = true
|
||||
delete(p.said, key+".raised")
|
||||
}
|
||||
for _, m := range t.Movers {
|
||||
if m.Tier <= t.Tier {
|
||||
continue
|
||||
}
|
||||
if end, ok := p.ends.endOf(m.Module, branch); ok && end.Pipeline == pipeline {
|
||||
return // a later tier's mover still kept at this pipeline: its want stays unwritten
|
||||
}
|
||||
}
|
||||
if len(notRunning) == 0 {
|
||||
if err := p.ends.st.drop(tierKey(pipeline)); err != nil {
|
||||
p.logf("the tier progress of %s is not deleted: %v", pipeline, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tierLateFinding is the finding tier-late of pipeline's tier, naming the movers and machines not running (design 47 H §751).
|
||||
func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunning) assign.Observed {
|
||||
var named []string
|
||||
for _, nr := range notRunning {
|
||||
named = append(named, fmt.Sprintf("%s on %s", nr.Module, strings.Join(nr.Machines, ", ")))
|
||||
}
|
||||
summary := fmt.Sprintf("pipeline %s: tier %d does not run within tier-bound: %s", pipeline, tier, strings.Join(named, "; "))
|
||||
return assign.Observed{Scope: mcc.ScopeDelivery, ID: pipeline, Kind: kindTierLate, Severity: mcseat.FindingSeverityWarning,
|
||||
Summary: summary, Headline: "A delivery's tier does not run in time",
|
||||
Explanation: summary + ". The assigner writes no later tier of this pipeline; the delivery ends here.",
|
||||
Source: "assigner." + kindTierLate}
|
||||
}
|
||||
|
||||
// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its
|
||||
// revision the number of the pipeline's key `<branch>.<revision>`, its source the one the pipeline-ended named (design 47
|
||||
// H-A-214, H-A-216), the key's number for a progress kept before it held one.
|
||||
func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) {
|
||||
var revision int64
|
||||
_, _ = fmt.Sscan(number, &revision)
|
||||
if source <= 0 {
|
||||
source = revision
|
||||
}
|
||||
body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning}
|
||||
event := meshwants.EventPipelineWanted + "." + pipeline
|
||||
if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil {
|
||||
p.logf("pipeline-wanted of %s is not said: %v", pipeline, err)
|
||||
}
|
||||
}
|
||||
|
||||
// tierDeadline sets the one timer that wakes the pass at the earliest tier's deadline (H-A-56); zero stops it.
|
||||
func (p *passer) tierDeadline(at time.Time) {
|
||||
if p.tierWake == nil {
|
||||
return
|
||||
}
|
||||
if p.tierTimer != nil {
|
||||
p.tierTimer.Stop()
|
||||
}
|
||||
if at.IsZero() {
|
||||
return
|
||||
}
|
||||
p.tierTimer = time.AfterFunc(max(at.Sub(p.clock()), 0), p.tierWake)
|
||||
}
|
||||
|
||||
// tierBoundSetting is the setting tier-bound in minutes (design 47 H §751), read at the start from the file the manifest declares
|
||||
// with the content ${config:tier-bound}; zero, its default, for a file missing or empty.
|
||||
func tierBoundSetting(getenv func(string) string) (time.Duration, error) {
|
||||
minutes, err := settingFile(getenv("MESH_ASSIGNER_TIER_BOUND_FILE"))
|
||||
return time.Duration(minutes) * tierBoundUnit, err
|
||||
}
|
||||
|
||||
// tierBoundUnit is the unit of the setting tier-bound, a minute; the whole test sets a shorter one, so a tier passes its bound
|
||||
// without a test waiting minutes.
|
||||
var tierBoundUnit = time.Minute
|
||||
@@ -1,215 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A7/AC1, A7/AC2 (design 47 H §751, H-A-56): a pipeline of a provider (tier 1) and its consumer (tier 2) writes the provider's
|
||||
// wants first and the consumer's only once the provider runs at its fingerprint on every present placing machine (one machine
|
||||
// away is not waited for), then says pipeline-wanted running and drops its progress; a provider that never runs is late at
|
||||
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
|
||||
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
|
||||
// H-A-485: a pipeline started again at the head, keyed `<branch>.<revision>.again.<n>`, moves tier by tier as a take-in's does.
|
||||
for _, c := range []struct {
|
||||
runs bool
|
||||
pipeline string
|
||||
}{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} {
|
||||
runs, pipeline := c.runs, c.pipeline
|
||||
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
|
||||
p.now = func() time.Time { return now }
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
findings := map[string]mcseat.Finding{}
|
||||
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { findings[views.FindingKey(f.Scope, f.ID, f.Kind)] = f; return nil },
|
||||
Delete: func(key string) error { delete(findings, key); return nil }}
|
||||
var said []meshwants.PipelineWanted
|
||||
p.publishWants = func(event string, body any, _ string) error {
|
||||
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
|
||||
if event != "pipeline-wanted."+pipeline {
|
||||
t.Errorf("pipeline-wanted said as %s", event)
|
||||
}
|
||||
said = append(said, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for i, m := range []struct{ name, fp string }{{"db", db}, {"web", web}} {
|
||||
digest := strings.Repeat(string(rune('e'+i)), 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.fp, Manifest: digest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[digest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
}
|
||||
placeOn(t, p.w, "g14", "db", "web")
|
||||
placeOn(t, p.w, "ace", "db", "web")
|
||||
p.w.heardPresence("ace", true, start) // away: not waited for
|
||||
p.w.mu.Lock()
|
||||
p.w.runningHeard = true
|
||||
p.w.mu.Unlock()
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
|
||||
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
|
||||
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
|
||||
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
|
||||
e.Source, e.Pipeline = 4, pipeline
|
||||
if _, err := p.ends.take(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wanted := func(node, module string) bool {
|
||||
key, _ := mwc.WantKey(node, module)
|
||||
_, ok := wantsKV.values[key]
|
||||
return ok
|
||||
}
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values)
|
||||
}
|
||||
if runs {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
|
||||
if err := p.w.hearRunning("running.g14.db", raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = start.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "web") || !wanted("ace", "web") {
|
||||
t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values)
|
||||
}
|
||||
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
|
||||
_ = p.w.hearRunning("running.g14.web", raw)
|
||||
p.wants(context.Background())
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
|
||||
t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings)
|
||||
}
|
||||
if _, _, ok := p.ends.tiersOf(pipeline); ok {
|
||||
t.Error("the tier progress stands after every tier ran")
|
||||
}
|
||||
continue
|
||||
}
|
||||
now = start.Add(29 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
if len(findings) != 0 || len(said) != 0 {
|
||||
t.Fatalf("late before tier-bound: %v, %+v", findings, said)
|
||||
}
|
||||
now = start.Add(30 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"]
|
||||
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
|
||||
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
|
||||
}
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" ||
|
||||
said[0].Revision != 5 || said[0].Source != 4 {
|
||||
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
|
||||
}
|
||||
// A tier progress kept before it held its source says the key's number as both (H-A-216).
|
||||
p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
|
||||
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
|
||||
}
|
||||
if wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Error("a consumer's want is written after its provider's tier was late")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-270, H-A-273, H-A-277 (1): a pipeline's tier progress holds every passed mover, versioned or not, and a tier with a mover
|
||||
// whose version is not yet written does not run until it is: no later tier is wanted out of order. When the lowest tier's version
|
||||
// comes last, nothing is wanted before it; with db (1) and api (3) versioned and web (2) late, api is wanted only once web runs.
|
||||
func TestALateMoverHoldsItsTier(t *testing.T) {
|
||||
type step struct {
|
||||
versions map[string]string // a pipeline-ended taken, each mover's version ("": none yet)
|
||||
runs string // or: this module runs on g14
|
||||
wanted string // then: the modules wanted, in order of the list
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
steps []step
|
||||
}{
|
||||
{"the lowest tier's version comes last", []step{
|
||||
{versions: map[string]string{"db": "", "web": "0.1.2", "api": "0.3.0"}, wanted: ""},
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db"},
|
||||
{runs: "db", wanted: "db,web"},
|
||||
{runs: "web", wanted: "db,web,api"},
|
||||
}},
|
||||
{"a middle tier's version comes late", []step{
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "", "api": "0.3.0"}, wanted: "db"},
|
||||
{runs: "db", wanted: "db"},
|
||||
{versions: map[string]string{"db": "0.2.0", "web": "0.1.2", "api": "0.3.0"}, wanted: "db,web"},
|
||||
{runs: "web", wanted: "db,web,api"},
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fps := map[string]string{}
|
||||
tiers := map[string]int64{"db": 1, "web": 2, "api": 3}
|
||||
for i, m := range []string{"db", "web", "api"} {
|
||||
fps[m] = meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i)), 64)
|
||||
}
|
||||
start := time.Date(2026, 11, 21, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, start)
|
||||
p.now = func() time.Time { return now }
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(string, any, string) error { return nil }
|
||||
for i, m := range []string{"db", "web", "api"} {
|
||||
digest := strings.Repeat(string(rune('1'+i)), 64)
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fps[m], Manifest: digest, Repository: "novox/" + m, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, m, mod)
|
||||
p.w.manifests[digest] = parsed{module: m, manifest: catalogue.Manifest{Module: m}}
|
||||
}
|
||||
placeOn(t, p.w, "g14", "db", "web", "api")
|
||||
p.w.mu.Lock()
|
||||
p.w.runningHeard = true
|
||||
p.w.mu.Unlock()
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now }
|
||||
for i, s := range c.steps {
|
||||
if s.versions != nil {
|
||||
var movers []mpr.MoverEnded
|
||||
for _, m := range []string{"db", "web", "api"} {
|
||||
movers = append(movers, mover(m, fps[m], s.versions[m], tiers[m], mpr.StepResultOutcomePassed))
|
||||
}
|
||||
if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, movers...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: s.runs, Fingerprint: fps[s.runs]})
|
||||
if err := p.w.hearRunning("running.g14."+s.runs, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
var got []string
|
||||
for _, m := range []string{"db", "web", "api"} {
|
||||
key, _ := mwc.WantKey("g14", m)
|
||||
if _, ok := wantsKV.values[key]; ok {
|
||||
got = append(got, m)
|
||||
}
|
||||
}
|
||||
if strings.Join(got, ",") != s.wanted {
|
||||
t.Fatalf("step %d: wanted %v, want %q", i+1, got, s.wanted)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -17,20 +12,13 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
@@ -48,9 +36,7 @@ type store interface {
|
||||
|
||||
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||
type putTools struct {
|
||||
st store
|
||||
// ask asks a module's tool (stdio.Ask, or a test's): the indexer's first view of modules for newest.
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
st store
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||
@@ -58,8 +44,8 @@ type putTools struct {
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newPutTools(st store, ask func(string, any) (json.RawMessage, error)) *putTools {
|
||||
return &putTools{st: st, ask: ask, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
func newPutTools(st store) *putTools {
|
||||
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// hear takes a put's change in from the watch.
|
||||
@@ -71,66 +57,72 @@ func (s *putTools) hear(key string, value []byte) {
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fullFP is a fingerprint of the new form, whole.
|
||||
var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||
var (
|
||||
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||
)
|
||||
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181:
|
||||
// a put names the whole fingerprint, the builder's builds being no bucket to list).
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||
// of one the builder built, which must name one build alone.
|
||||
func (s *putTools) version(module, version string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(version))
|
||||
switch {
|
||||
case v == "newest":
|
||||
// Asked of the indexer's first view of modules, never read from its bus state (ADR 0596, ADR 0621; issue 1966).
|
||||
mods, err := indexerView(s.ask, indexerModules)
|
||||
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
raw, found := mods[module]
|
||||
if !found {
|
||||
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||
}
|
||||
m, err := meshcatalogclient.DecodeModule(raw)
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if m.Fp2 == "" {
|
||||
if m.FP2 == "" {
|
||||
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||
}
|
||||
return m.Fp2, nil
|
||||
return m.FP2, nil
|
||||
case fullFP.MatchString(v):
|
||||
return v, nil
|
||||
case prefix.MatchString(v):
|
||||
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||
found = append(found, fp)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
sort.Strings(found)
|
||||
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||
}
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version)
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||
}
|
||||
|
||||
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||
type putAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
puts.Put
|
||||
delivery.Put
|
||||
Follow string `json:"follow,omitempty"`
|
||||
// Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the
|
||||
// node's want until it ends or is released.
|
||||
Older string `json:"older-than-newest,omitempty"`
|
||||
// Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is
|
||||
// written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*).
|
||||
Since time.Time `json:"since"`
|
||||
Bound string `json:"bound,omitempty"`
|
||||
}
|
||||
|
||||
// answerOf is what assigner_put and assigner_puts answer of the put p of module on node.
|
||||
func answerOf(node, module string, p puts.Put) putAnswer {
|
||||
a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()}
|
||||
if putWaits(p) {
|
||||
a.Bound = putWaitBound.String()
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// doPut is the tool assigner_put.
|
||||
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := puts.Key(node, module)
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -154,17 +146,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
} else if found {
|
||||
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
|
||||
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||
if before.FP == fp && before.Hold == hold {
|
||||
a := answerOf(node, module, before)
|
||||
a.Follow = followPut(node, module)
|
||||
return a, nil
|
||||
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||
}
|
||||
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||
}
|
||||
}
|
||||
p := puts.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, puts.Asked, words)
|
||||
data, err := puts.Encode(p)
|
||||
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -172,21 +162,19 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
a := answerOf(node, module, s.await(key, p))
|
||||
a.Follow = followPut(node, module)
|
||||
return a, nil
|
||||
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||
}
|
||||
|
||||
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||
func (s *putTools) await(key string, p puts.Put) puts.Put {
|
||||
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||
deadline := time.After(answerWithin)
|
||||
for {
|
||||
s.mu.Lock()
|
||||
raw, changed := s.heard[key], s.changed
|
||||
s.mu.Unlock()
|
||||
if now, err := puts.Decode(raw); err == nil && now.At.Equal(p.At) {
|
||||
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||
p = now
|
||||
if now.Step != puts.Asked && now.Step != puts.Build && now.Step != puts.Compose {
|
||||
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||
return p // it waits on a person or the node, or ended
|
||||
}
|
||||
}
|
||||
@@ -206,11 +194,8 @@ func followPut(node, module string) string {
|
||||
|
||||
// doRelease is the tool assigner_release.
|
||||
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
if arg(args, "how") != "" {
|
||||
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
|
||||
}
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := puts.Key(node, module)
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -224,15 +209,15 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||
}
|
||||
p, err := puts.Decode(raw)
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if puts.Ended(p.Step) {
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||
}
|
||||
p = p.Said(s.now(), puts.Released, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := puts.Encode(p)
|
||||
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -240,7 +225,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return answerOf(node, module, p), nil
|
||||
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||
}
|
||||
|
||||
// doPuts is the tool assigner_puts.
|
||||
@@ -251,10 +236,8 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []putAnswer{}
|
||||
newest := map[string]string{} // the indexer's newest fingerprint by module, asked once and only for a put that stands
|
||||
asked := false
|
||||
for _, key := range keys {
|
||||
node, module, err := puts.ParseKey(key)
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
@@ -262,27 +245,11 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := puts.Decode(raw)
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
a := answerOf(node, module, p)
|
||||
if !puts.Ended(p.Step) {
|
||||
if !asked {
|
||||
asked = true
|
||||
if mods, err := indexerView(s.ask, indexerModules); err == nil {
|
||||
for name, raw := range mods {
|
||||
if m, err := meshcatalogclient.DecodeModule(raw); err == nil {
|
||||
newest[name] = m.Fp2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if n := newest[module]; n != "" && n != p.FP {
|
||||
a.Older = n
|
||||
}
|
||||
}
|
||||
out = append(out, a)
|
||||
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||
}
|
||||
return map[string]any{"puts": out}, nil
|
||||
}
|
||||
@@ -339,12 +306,9 @@ func (s *putTools) tools() []stdio.Tool {
|
||||
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
|
||||
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
|
||||
"failed (the failing check is the delivery's), or rejudge (judge it again now); written in the hand-act log first"},
|
||||
[]string{"module", "why"}),
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||
[]string{"node", "module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
s.waitsTool(),
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
@@ -372,8 +336,7 @@ func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(st
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||
func serveTools() error {
|
||||
plc.SetModule(os.Getenv(stdio.ServedAs)) // the module the runtime serves these tools as (A-296)
|
||||
s := newPutTools(runtimeStore{}, stdio.Ask)
|
||||
s := newPutTools(runtimeStore{})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
@@ -388,44 +351,17 @@ func serveTools() error {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
// The seat list (seatlist.go), read through the runtime once Serve has its channel: the seat's verbs judge by it.
|
||||
gate := seatlist.NewPart("assigner")
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: "+f+"\n", a...) })
|
||||
}()
|
||||
seatVerbs := newSeatTools(runtimeStore{}, stdio.Ask)
|
||||
seatVerbs.seatList = gate
|
||||
tools := append(s.tools(), seatVerbs.tools()...)
|
||||
tools = append(tools, gate.Tools()...)
|
||||
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
|
||||
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
|
||||
tools = append(tools, newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
|
||||
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
|
||||
return stdio.SeatPublish(wantsEventSubject(e), b, id)
|
||||
})
|
||||
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
|
||||
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
|
||||
seatVerbs.watched = wants.watched
|
||||
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
|
||||
tools = append(tools, wants.tools()...)
|
||||
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
|
||||
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
|
||||
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// What each node's engine reads, as the assigner uses it: its own state engine-reads (design 47 H-A-104).
|
||||
tools = append(tools, engineReadsTools(runtimeStore{})...)
|
||||
tools = append(tools, writtenTools(runtimeStore{})...)
|
||||
tools = append(tools, whyTools(runtimeStore{})...)
|
||||
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
|
||||
tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...)
|
||||
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777)
|
||||
tools = append(tools, keepNowTools(stdio.Ask)...)
|
||||
tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2))
|
||||
}).tools()...)
|
||||
return stdio.Serve("", tools)
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
func shortFP(fp string) string {
|
||||
part, err := catalogkey.KeyPart(fp)
|
||||
part, err := delivery.FingerprintKeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
return fp
|
||||
}
|
||||
|
||||
@@ -1,493 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat
|
||||
// mesh-wants, asked by the runner for each mover before its composition step. One run per take-in revision, a key
|
||||
// `<pipeline>.validate` of the state composition-runs in the CompositionRun form with no machine, which every mover's step
|
||||
// reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from
|
||||
// the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the
|
||||
// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input.
|
||||
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case
|
||||
// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed.
|
||||
|
||||
const (
|
||||
// validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's
|
||||
// cmd/mesh-judge, design 47 H-A-34).
|
||||
validatorPackage = "mesh-delivery-indexer/cmd/mesh-judge"
|
||||
// validateKeySuffix ends the key of a revision's validate run, `<pipeline>.validate` (H-A-108).
|
||||
validateKeySuffix = "." + meshwants.StepRunStepValidate
|
||||
// beforeCase is the validator's case of the revision's base, beside changeCase (H-A-108 (3), H-A-151, H-A-154).
|
||||
beforeCase = "before"
|
||||
// The reasons a validate step fails with, the first word of its said (§745).
|
||||
reasonManifestRefused = "manifest-refused"
|
||||
reasonCycle = "cycle"
|
||||
// betweenManifests is the validator's name for what it judges between manifests (the catalogue as a whole).
|
||||
betweenManifests = "(between manifests)"
|
||||
)
|
||||
|
||||
// validateKeyOf is the key of the validate run of the step's take-in revision.
|
||||
func validateKeyOf(step meshwants.StepRun) (string, error) {
|
||||
if step.Pipeline == "" {
|
||||
return "", errors.New("the step names no pipeline, so no validate run is named")
|
||||
}
|
||||
return step.Pipeline + validateKeySuffix, nil
|
||||
}
|
||||
|
||||
// isValidateRun says whether a run of the state is a revision's validate run: one with no machine.
|
||||
func isValidateRun(run meshwants.CompositionRun) bool { return run.Machine == "" }
|
||||
|
||||
// validate runs one mover's validate step: it joins or starts the revision's validate run and judges the step from its end.
|
||||
func (r *compositionRun) validate(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
j, err := r.runs.joinAt(key, readerOf(id, step), candidate, "", buildstep.NewID())
|
||||
if err != nil {
|
||||
return couldNotRun("the validate run " + key + " is not joined: " + err.Error())
|
||||
}
|
||||
if j.started {
|
||||
go r.startValidate(ctx, key, j.run, step)
|
||||
}
|
||||
return r.readValidate(ctx, key, step)
|
||||
}
|
||||
|
||||
// resumeValidate judges a validate step taken up at the assigner's start from its revision's run.
|
||||
func (r *compositionRun) resumeValidate(ctx context.Context, _ string, step meshwants.StepRun) stepOutcome {
|
||||
key, err := validateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
return r.readValidate(ctx, key, step)
|
||||
}
|
||||
|
||||
// readValidate waits for the end of the validate run of key and judges the step from it.
|
||||
func (r *compositionRun) readValidate(ctx context.Context, key string, step meshwants.StepRun) stepOutcome {
|
||||
setShared(ctx, r.runs.sharedOf([]string{key}, nil))
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
run := r.runs.await(ctx, []string{key})[key]
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
|
||||
out := r.judgeValidate(ctx, step, run)
|
||||
out.Shared = r.runs.sharedOf([]string{key}, map[string]meshwants.CompositionRun{key: run})
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeValidate is a step's end from its revision's validate run: could not run with the run's words; failed `cycle` naming
|
||||
// the cycle; failed `manifest-refused` naming every problem the revision causes; else passed.
|
||||
func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepRun, run meshwants.CompositionRun) stepOutcome {
|
||||
if run.Ended == nil {
|
||||
return couldNotRun("the step stopped before its validate run ended")
|
||||
}
|
||||
if run.Outcome != meshwants.CompositionRunOutcomePassed {
|
||||
return couldNotRun(run.Said)
|
||||
}
|
||||
var v validateVerdict
|
||||
if raw, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(raw, &v) != nil {
|
||||
return couldNotRun("the validate run's verdict does not read")
|
||||
}
|
||||
if len(v.Cycle) > 0 {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: reasonCycle + ": " + cycleWords(v.Cycle)}
|
||||
}
|
||||
half, err := r.revision(ctx, step)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's catalog revision is not read: " + err.Error())
|
||||
}
|
||||
// A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151).
|
||||
empty := ""
|
||||
if half.Revision.Base <= 0 {
|
||||
empty = emptyBeforeWords
|
||||
}
|
||||
if problems := causedProblems(v.Change, v.Before); len(problems) > 0 {
|
||||
said := reasonManifestRefused + ": " + strings.Join(problems, "; ")
|
||||
if empty != "" {
|
||||
said += " (" + empty + ")"
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: said}
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed, Said: empty}
|
||||
}
|
||||
|
||||
// emptyBeforeWords say, in a validate step's end, that its revision names no base: the branch's first take-in, judged against
|
||||
// an empty case before (H-A-151, H-A-154).
|
||||
const emptyBeforeWords = "the case before is empty: the revision names no base, the branch's first take-in"
|
||||
|
||||
// validateVerdict is a validate run's verdict as the run keeps it: the cycle the revision makes, or the validator's two cases.
|
||||
type validateVerdict struct {
|
||||
Cycle []meshcatalog.ArtifactRef `json:"cycle,omitempty"`
|
||||
Change composer.CaseVerdict `json:"change"`
|
||||
Before composer.CaseVerdict `json:"before"`
|
||||
}
|
||||
|
||||
// asVerdict is v as a run's verdict.
|
||||
func (v validateVerdict) asVerdict() (map[string]any, error) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]any
|
||||
return out, json.Unmarshal(raw, &out)
|
||||
}
|
||||
|
||||
// cycleWords names a cycle as the artifacts it passes through.
|
||||
func cycleWords(cycle []meshcatalog.ArtifactRef) string {
|
||||
words := make([]string, len(cycle))
|
||||
for i, a := range cycle {
|
||||
words[i] = string(a)
|
||||
}
|
||||
return strings.Join(words, " → ")
|
||||
}
|
||||
|
||||
// before is the case before of the step's revision half: the revision its base names, the newest kept revision below it that
|
||||
// its take-in pinned, read through the seat mesh-catalog's verb revision (H-A-108 (3), H-A-151, H-A-154); empty where the
|
||||
// revision names no base, the branch's first take-in.
|
||||
func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, half mcc.Half) (mcc.Half, error) {
|
||||
if half.Revision.Base <= 0 {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}, nil
|
||||
}
|
||||
base := step
|
||||
base.Revision = half.Revision.Base
|
||||
return r.revision(ctx, base)
|
||||
}
|
||||
|
||||
// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between
|
||||
// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the
|
||||
// revision alters that manifest or not (assigner_validate_before answers those).
|
||||
func causedProblems(change, before composer.CaseVerdict) []string {
|
||||
return problemsBy(change, before, false)
|
||||
}
|
||||
|
||||
// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814).
|
||||
func uncountedProblems(change, before composer.CaseVerdict) []string {
|
||||
return problemsBy(change, before, true)
|
||||
}
|
||||
|
||||
// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had).
|
||||
func problemsBy(change, before composer.CaseVerdict, had bool) []string {
|
||||
var out []string
|
||||
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
|
||||
read := change.Manifests[module]
|
||||
if read.Accepted || read.Refused == "" {
|
||||
continue
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, p := range strings.Split(before.Manifests[module].Refused, "\n") {
|
||||
held[p] = true
|
||||
}
|
||||
var named []string
|
||||
for _, p := range strings.Split(read.Refused, "\n") {
|
||||
if p != "" && held[p] == had {
|
||||
named = append(named, p)
|
||||
}
|
||||
}
|
||||
if len(named) == 0 {
|
||||
continue
|
||||
}
|
||||
what := module
|
||||
if module == betweenManifests {
|
||||
what = "between manifests"
|
||||
}
|
||||
out = append(out, what+": "+strings.Join(named, "; "))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// catalogEntry is the entry of the module that holds the seat mesh-catalog, or false.
|
||||
func catalogEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
|
||||
for _, name := range slices.Sorted(maps.Keys(entries)) {
|
||||
for _, s := range entries[name].Serves {
|
||||
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), meshcatalog.Seat+".") {
|
||||
return entries[name], true
|
||||
}
|
||||
}
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, false
|
||||
}
|
||||
|
||||
// manifestsOf are the half's manifests whole, as the validator's case holds them.
|
||||
func manifestsOf(half mcc.Half) (map[string]composer.Manifest, error) {
|
||||
out := map[string]composer.Manifest{}
|
||||
for name, e := range half.Entries {
|
||||
if len(e.Manifest) == 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
|
||||
}
|
||||
out[name] = composer.Manifest{Raw: raw}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// startValidate runs the validate run the step started: the revision's cycle ends it at once; else the validator's request
|
||||
// is kept once in `inputs`, its program and input recorded on the run, and one run ask put on the Go build seat. A fault ends
|
||||
// the run could not run.
|
||||
func (r *compositionRun) startValidate(ctx context.Context, key string, run meshwants.CompositionRun, step meshwants.StepRun) {
|
||||
ask := run.Ask
|
||||
end := func(outcome, said string, verdict map[string]any) {
|
||||
if _, err := r.runs.endRun(key, ask, outcome, said, verdict); err != nil {
|
||||
r.logf("the validate run %s is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
couldNot := func(said string) { end(meshwants.CompositionRunOutcomeCouldNotRun, said, nil) }
|
||||
half, err := r.revision(ctx, step)
|
||||
if err != nil {
|
||||
couldNot("the step's catalog revision is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if cycle := graph.RefusedCycle(half.Entries); cycle != nil {
|
||||
verdict, err := validateVerdict{Cycle: cycle}.asVerdict()
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
end(meshwants.CompositionRunOutcomePassed, "", verdict)
|
||||
return
|
||||
}
|
||||
before, err := r.before(ctx, step, half)
|
||||
if err != nil {
|
||||
couldNot("the base of the case before is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
entry, ok := catalogEntry(half.Entries)
|
||||
if !ok {
|
||||
couldNot(fmt.Sprintf("the catalog revision %d holds no module claiming the seat %s, so no manifest validator is built", half.Revision.Revision, meshcatalog.Seat))
|
||||
return
|
||||
}
|
||||
program, err := programOfEntry(entry)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
program.Package = validatorPackage
|
||||
seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
change, err := manifestsOf(half)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
prior, err := manifestsOf(before)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
// No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)).
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}},
|
||||
Seats: seats, SeatList: list}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
couldNot("its request cannot be encoded: " + err.Error())
|
||||
return
|
||||
}
|
||||
digest, err := r.keeper.keep(ask, raw)
|
||||
if err != nil {
|
||||
couldNot("its request is not kept: " + err.Error())
|
||||
return
|
||||
}
|
||||
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
|
||||
if err != nil {
|
||||
couldNot(err.Error())
|
||||
return
|
||||
}
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: entry.Fp}
|
||||
c.Input = digest
|
||||
return true
|
||||
}); err != nil {
|
||||
couldNot("the run's program and input are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the manifest validator of " + step.Pipeline + " at " + program.Commit},
|
||||
Packages: pinsOfEntry(entry)} // the validator is built against its entry's pins at the step's revision (H-A-203, H-A-192, H-A-205)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
couldNot("its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// validateRunVerdict reads the validator's answer: its two cases as the run keeps them, or why it is none.
|
||||
func validateRunVerdict(answer []byte) (map[string]any, string) {
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(answer, &v); err != nil {
|
||||
return nil, "the manifest validator answered no verdict: " + err.Error()
|
||||
}
|
||||
if v.V != composer.Version {
|
||||
return nil, fmt.Sprintf("the manifest validator answered a verdict of version %d, the assigner reads %d", v.V, composer.Version)
|
||||
}
|
||||
var out validateVerdict
|
||||
found := 0
|
||||
for _, c := range v.Cases {
|
||||
switch c.Name {
|
||||
case changeCase:
|
||||
out.Change, found = c, found+1
|
||||
case beforeCase:
|
||||
out.Before, found = c, found+1
|
||||
}
|
||||
}
|
||||
if found != 2 {
|
||||
return nil, "the manifest validator's verdict does not hold both the cases " + changeCase + " and " + beforeCase
|
||||
}
|
||||
verdict, err := out.asVerdict()
|
||||
if err != nil {
|
||||
return nil, "the manifest validator's verdict cannot be kept: " + err.Error()
|
||||
}
|
||||
return verdict, ""
|
||||
}
|
||||
|
||||
// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814).
|
||||
const validateBeforeTool = "assigner_validate_before"
|
||||
|
||||
// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too,
|
||||
// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps.
|
||||
func validateBefore(st store, pipeline, module string) (map[string]any, error) {
|
||||
key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, ok, err := st.get(buckets.CompositionRunsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no validate run %s is kept", key)
|
||||
}
|
||||
var run meshwants.CompositionRun
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
return nil, fmt.Errorf("the validate run %s does not read: %w", key, err)
|
||||
}
|
||||
var v validateVerdict
|
||||
if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil {
|
||||
return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key)
|
||||
}
|
||||
change, before := v.Change, v.Before
|
||||
if module = strings.TrimSpace(module); module != "" {
|
||||
change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}}
|
||||
}
|
||||
uncounted := uncountedProblems(change, before)
|
||||
if uncounted == nil {
|
||||
uncounted = []string{}
|
||||
}
|
||||
return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil
|
||||
}
|
||||
|
||||
// validateBeforeTools are the module's tool of the validate step's before case.
|
||||
func validateBeforeTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " +
|
||||
"validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " +
|
||||
"for one module when named. Read-only.",
|
||||
Input: schema(map[string]string{"pipeline": "the pipeline's key, <branch>.<revision>", "module": "one module; absent: every one"}, []string{"pipeline"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return validateBefore(st, arg(args, "pipeline"), arg(args, "module"))
|
||||
}}}
|
||||
}
|
||||
|
||||
// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443).
|
||||
const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList
|
||||
|
||||
// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by
|
||||
// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them,
|
||||
// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb
|
||||
// seat-list hands, asked as a kept call under the run's ask.
|
||||
func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) {
|
||||
if len(defs) == 0 {
|
||||
return r.seatList(ask)
|
||||
}
|
||||
sorted := slices.Clone(defs)
|
||||
slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) })
|
||||
seats, err := r.seats(ctx, sorted)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
raw, err := json.Marshal(seats)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err)
|
||||
}
|
||||
var list []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &list); err != nil {
|
||||
return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err)
|
||||
}
|
||||
return list, seatListDigest(raw), nil
|
||||
}
|
||||
|
||||
// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex.
|
||||
func seatListDigest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in
|
||||
// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest
|
||||
// judges the same bytes.
|
||||
func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) {
|
||||
what := "the trunk's seat list is not read through " + seatListKey
|
||||
if r.callFile == nil {
|
||||
return nil, "", errors.New(what + ": the assigner holds no chunked call")
|
||||
}
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
return nil, "", fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey,
|
||||
meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])})
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
var a struct {
|
||||
meshcatalog.SeatListAnswer
|
||||
File string `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err)
|
||||
}
|
||||
if a.Digest == "" {
|
||||
return nil, "", errors.New(what + ": its answer names no digest")
|
||||
}
|
||||
file, err := base64.StdEncoding.DecodeString(a.File)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err)
|
||||
}
|
||||
if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest {
|
||||
return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file),
|
||||
seatListDigest(file), a.Size, a.Digest)
|
||||
}
|
||||
var list []json.RawMessage
|
||||
if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 {
|
||||
return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest)
|
||||
}
|
||||
return list, a.Digest, nil
|
||||
}
|
||||
@@ -1,220 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate
|
||||
// run, `<pipeline>.validate` of the state composition-runs; a manifest the revision alters that names an unknown field fails
|
||||
// it `manifest-refused` naming the problem, a problem of a manifest it leaves as it was does not, and the validator is the
|
||||
// revision's own entry of the module holding the seat mesh-catalog, on the Go build seat; a revision whose entries make a cycle
|
||||
// fails it `cycle` naming the cycle, with no validator asked. The case before is the revision its base names (H-A-151, H-A-154:
|
||||
// the newest kept below it, which the stand-in names as the indexer does); a revision naming no base, the branch's first take-in,
|
||||
// is judged against an empty case before, said in its end, and no revision below it is asked.
|
||||
func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix",
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient": "v0.1.4"}}
|
||||
entry := func(module, digest string, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
revisions := map[int64][]meshcatalog.CatalogEntry{
|
||||
6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")},
|
||||
7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")},
|
||||
3: {indexer},
|
||||
12: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
}
|
||||
// base is the newest kept revision below n, as the take-in names it; none below the first. Revision 12 names 7 with 8 to 11
|
||||
// never kept, so a case before read as the number minus one could not run (H-A-154).
|
||||
base := func(n int64) (b int64) {
|
||||
if n == 12 {
|
||||
return 7
|
||||
}
|
||||
for k := range revisions {
|
||||
if k < n && k > b {
|
||||
b = k
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
switch key {
|
||||
case mcc.RevisionVerb:
|
||||
var in meshcatalog.RevisionInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
entries, ok := revisions[in.Revision]
|
||||
if !ok || in.Branch != "main" {
|
||||
return nil, errors.New("asked another revision")
|
||||
}
|
||||
// Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)).
|
||||
var defs []meshcatalog.CatalogRevisionDefinitionsItem
|
||||
if in.Revision != 3 {
|
||||
defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision),
|
||||
Definitions: defs}, Entries: entries})
|
||||
case seatDefinitionsKey:
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}})
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
// The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file.
|
||||
trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`)
|
||||
listAnswer := func(file []byte, digest string, size int) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)})
|
||||
return raw
|
||||
}
|
||||
var listAsks []process.KeptAsk
|
||||
listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
|
||||
callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
in, _ := body.(meshcatalog.SeatListInput)
|
||||
if key != seatListKey || len(in.Call) != 32 {
|
||||
return nil, errors.New("asked " + key + " with the call " + in.Call)
|
||||
}
|
||||
listAsks = append(listAsks, ask)
|
||||
return listed, nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var asks []buildstep.StepAsk
|
||||
var requests []composer.Request
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
|
||||
r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf,
|
||||
put: func(_ context.Context, a buildstep.StepAsk) error {
|
||||
body, err := keeper.body(a.Run.Input.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := composer.Decode(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mu.Lock()
|
||||
asks, requests = append(asks, a), append(requests, req)
|
||||
mu.Unlock()
|
||||
answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{
|
||||
{Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}},
|
||||
{Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}})
|
||||
go func() {
|
||||
if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"},
|
||||
func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}}
|
||||
steps := &compositionSteps{validate: r.validate, run: r.run}
|
||||
if steps.howOf(meshwants.StepRunStepValidate, false) == nil || steps.howOf("check", false) != nil {
|
||||
t.Error("the validate step is not taken, or another step is")
|
||||
}
|
||||
step := func(module string, revision int64) meshwants.StepRun {
|
||||
return meshwants.StepRun{Step: meshwants.StepRunStepValidate, Pipeline: "main." + string(rune('0'+revision)), Branch: "main", Module: module,
|
||||
Revision: revision, Source: revision}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
outs := make([]stepOutcome, 2)
|
||||
var wg sync.WaitGroup
|
||||
for i, module := range []string{"web", "db"} {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); outs[i] = r.validate(ctx, "ask-"+module, step(module, 7)) }()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) ||
|
||||
strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems
|
||||
t.Errorf("mover %d's validate step ended %+v", i, o)
|
||||
}
|
||||
}
|
||||
// assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count.
|
||||
runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}}
|
||||
kept, _, _, _ := runs.read("main.7" + validateKeySuffix)
|
||||
runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept)
|
||||
before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"})
|
||||
if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) {
|
||||
t.Errorf("assigner_validate_before answered %+v, %v", before, err)
|
||||
}
|
||||
if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil {
|
||||
t.Error("assigner_validate_before answers a run not kept")
|
||||
}
|
||||
if len(asks) != 1 {
|
||||
t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks))
|
||||
}
|
||||
if p := asks[0].Run.Program; p.Commit != commitE3 || p.Dir != "modules/mesh-delivery-indexer" || p.Package != validatorPackage || len(asks[0].Run.Heads) != 0 {
|
||||
t.Errorf("the validator is built from %+v", asks[0].Run)
|
||||
}
|
||||
// The validator is built against the indexer entry's pins at the step's revision (H-A-203, H-A-192, H-A-205).
|
||||
if got := asks[0].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"] != "v0.1.4" {
|
||||
t.Errorf("the validator is built against the pins %v; want its entry's", got)
|
||||
}
|
||||
if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil {
|
||||
t.Errorf("the validator's request is %+v", requests[0])
|
||||
}
|
||||
// A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat
|
||||
// list is asked.
|
||||
if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 {
|
||||
t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own))
|
||||
}
|
||||
if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok {
|
||||
t.Error("the validate run is not the key <pipeline>.validate of composition-runs")
|
||||
}
|
||||
o := r.validate(ctx, "ask-a", step("a", 9))
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || o.Said != reasonCycle+": a:x → b:y → a:x" || len(asks) != 1 {
|
||||
t.Errorf("a revision making a cycle ended %+v (%d validator asks)", o, len(asks))
|
||||
}
|
||||
// The stand-in validator answers its one verdict: judged against the empty case before, web's problem is the revision's.
|
||||
if o := r.validate(ctx, "ask-first", step("indexer", 3)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 {
|
||||
t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks))
|
||||
}
|
||||
// Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest,
|
||||
// asked once as a kept call under the run's ask (H-A-441 (b), H-A-443).
|
||||
if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) ||
|
||||
len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 {
|
||||
t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList))
|
||||
}
|
||||
// Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
answer json.RawMessage
|
||||
}{
|
||||
{"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))},
|
||||
{"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)},
|
||||
{"no digest", listAnswer(trunkList, "", len(trunkList))},
|
||||
{"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))},
|
||||
{"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)},
|
||||
} {
|
||||
listed = row.answer
|
||||
if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) {
|
||||
t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err)
|
||||
}
|
||||
}
|
||||
listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
|
||||
// Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one
|
||||
// is not, and the case before holds base 7's manifests.
|
||||
if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour"` || len(asks) != 3 {
|
||||
t.Errorf("a revision naming base 7 below its number ended %+v (%d validator asks), want it judged against base 7", o, len(asks))
|
||||
} else if web := requests[2].Cases[1].Manifests["web"]; len(requests[2].Cases[1].Manifests) != 3 || !strings.Contains(string(web.Raw), "d-web-2") {
|
||||
t.Errorf("the case before of revision 12 is %+v, want base 7's manifests", requests[2].Cases[1].Manifests)
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
|
||||
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
|
||||
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
|
||||
// manifests, as a provider is counted (ADR 0603 §5).
|
||||
|
||||
// moduleGraph is the module graph of the catalog half of branch, from the entries' invokes and serves (design 47 G §184, §195).
|
||||
func (p *passer) moduleGraph(branch string) graph.Graph { return p.w.graphOf(branch) }
|
||||
|
||||
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
|
||||
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
|
||||
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
|
||||
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
|
||||
// a verb.
|
||||
func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
if len(g) == 0 {
|
||||
return nil
|
||||
}
|
||||
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if p.unread.changed(unread) && len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// saidSet is the last set of names said, so a pass says a set again only when it changed.
|
||||
type saidSet struct {
|
||||
mu sync.Mutex
|
||||
last string
|
||||
}
|
||||
|
||||
// changed keeps names as the set said last and says whether it differs from the one before.
|
||||
func (s *saidSet) changed(names []string) bool {
|
||||
sorted := slices.Sorted(slices.Values(names))
|
||||
key := strings.Join(sorted, " ")
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if key == s.last {
|
||||
return false
|
||||
}
|
||||
s.last = key
|
||||
return true
|
||||
}
|
||||
|
||||
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
|
||||
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
|
||||
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
|
||||
// that build serves what it offers (A-352); a consumer needing the newer build's forms waits by its kind-4 edge instead. A
|
||||
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
|
||||
// holds a consumer for good.
|
||||
func (p *passer) offering(trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
|
||||
offered := map[string]bool{}
|
||||
for node, haves := range n.Haves {
|
||||
if !n.Online[node] {
|
||||
continue
|
||||
}
|
||||
for module, h := range haves {
|
||||
if h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseWaitingYes || h.Health != nodeengineclient.HealthOK {
|
||||
continue
|
||||
}
|
||||
m, ok := p.manifestOf(module, h.Fp)
|
||||
if !ok {
|
||||
m = trunk[module]
|
||||
}
|
||||
for _, o := range m.Offers() {
|
||||
offered[o] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return offered[provision] }
|
||||
}
|
||||
|
||||
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
|
||||
func (p *passer) servesOf(module, fp string) ([]string, bool) {
|
||||
m, ok := p.manifestOf(module, fp)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return graph.ServesOf(m), true
|
||||
}
|
||||
|
||||
// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests
|
||||
// hold), and whether it could be read.
|
||||
func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
|
||||
e, ok := p.w.ends.endOf(module, fp)
|
||||
if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
got := p.w.manifest(e.Manifest)
|
||||
if got.err != "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
return got.manifest, true
|
||||
}
|
||||
|
||||
// servingHaves are the haves rule 8 counts (design 47 G A-426, A-426a): a node that runs a healthy build of a part serves what that
|
||||
// build serves, whatever its version and whether or not the catalog lists it, so a have held (its build has no catalog entry, a
|
||||
// put holds it) or waiting for a person's yes, whose build runs healthy, counts as running; a running statement that names a build
|
||||
// stopped, or not healthy, does not. counted is told each build so counted that its phase alone would have left out.
|
||||
func servingHaves(n assign.Nodes, counted func(node, module string, h nodeengine.HaveByNodeModuleValue)) map[string]map[string]nodeengine.HaveByNodeModuleValue {
|
||||
out := map[string]map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for node, haves := range n.Haves {
|
||||
out[node] = map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for module, h := range haves {
|
||||
if (h.Phase == nodeengineclient.PhaseHeld || h.Phase == nodeengineclient.PhaseWaitingYes) && h.Health == nodeengineclient.HealthOK {
|
||||
if r, ok := n.Running[node][module]; !ok || (!r.Stopped && (r.Healthy == nil || *r.Healthy)) {
|
||||
if counted != nil {
|
||||
counted(node, module, h)
|
||||
}
|
||||
h.Phase = nodeengineclient.PhaseRunning
|
||||
}
|
||||
}
|
||||
out[node][module] = h
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countedServing says once per node, module and build that rule 8 counted a build whose have is not in phase running as serving
|
||||
// its verbs (A-426a): the rule says which build it counted.
|
||||
func (p *passer) countedServing(node, module string, h nodeengine.HaveByNodeModuleValue) {
|
||||
key := "serving " + node + "." + module + "@" + h.Fp + "." + h.Phase
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
p.logf("%s on %s runs %s healthy in phase %s: counted as serving what that build serves, whatever its version (design 47 G A-426a)", module, node, shortHex(h.Fp), h.Phase)
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// marksBucket is the assigner's state of what its views family said each state up to.
|
||||
func marksBucket() string { return assignment.Module() + "_" + views.MarksState }
|
||||
|
||||
// viewedTool is one tool over one state of the assigner's.
|
||||
type viewedTool struct{ tool, state, words string }
|
||||
|
||||
// firstViewTools are the assigner's first-view tools (ADR 0621 §1): each answers only its callers list
|
||||
// (views.CallersEnv), which the manifest writes.
|
||||
var firstViewTools = []viewedTool{
|
||||
{"assignments", mma.StateAssignments, "where each module is placed: the seat mesh-module-assigner's assignments, by module."},
|
||||
{"seat-holders", mma.StateSeatHolders, "who holds which seat on which node."},
|
||||
{"wants", "wants", "what each node should run: want.<node>.<module>, its fingerprint and the rule that gave it."},
|
||||
// The seat mesh-wants' asks (design 47 G §717): the asks the assigner put to a person about what a node runs.
|
||||
{"asks", "asks", "the asks the assigner put to a person about what a node runs, and their answers."},
|
||||
}
|
||||
|
||||
// readTools are the assigner's ordinary read tools, one per state its readers read (design 25 § 12.2).
|
||||
var readTools = []viewedTool{
|
||||
{"assigner_read_assignments", mma.StateAssignments, "where each module is placed, by module."},
|
||||
{"assigner_read_seat_holders", mma.StateSeatHolders, "who holds which seat on which node."},
|
||||
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
|
||||
{"assigner_read_findings", assign.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
|
||||
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
|
||||
{"assigner_read_copies", copiesState, "the copies the assigner made of a retired state: when the copy into assignments ended and how many keys it copied."},
|
||||
}
|
||||
|
||||
// viewedStates are the states the views family says, each by its bucket: the six its readers read.
|
||||
func viewedStates() map[string]string {
|
||||
return map[string]string{
|
||||
mma.StateAssignments: plc.AssignmentsBucket(),
|
||||
mma.StateSeatHolders: plc.SeatHoldersBucket(),
|
||||
"wants": buckets.Wants(),
|
||||
assign.FindingsState: assign.FindingsBucket(),
|
||||
"asks": buckets.Asks(),
|
||||
"differences": buckets.Differences(),
|
||||
copiesState: copiesBucket(),
|
||||
}
|
||||
}
|
||||
|
||||
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
|
||||
// runtime of its own is connected (ADR 0621 §3).
|
||||
func viewTools(getenv func(string) string, state func(string) views.Source, now func() time.Time) []stdio.Tool {
|
||||
marks := state(views.MarksState)
|
||||
var out []stdio.Tool
|
||||
for _, t := range firstViewTools {
|
||||
out = append(out, views.FirstViewTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
|
||||
}
|
||||
for _, t := range readTools {
|
||||
out = append(out, views.ReadTool(assignment.Module(), t.tool, t.state, t.words, state(t.state), marks, now))
|
||||
}
|
||||
// Its own first views, of the indexer's manifests, modules and graph (DLV-F3r, ADR 0631 §2): read, and asked again by hand.
|
||||
return append(out, views.StatusTools(assignment.Module(), "assigner_first_views", "assigner_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, now)...)
|
||||
}
|
||||
|
||||
func keptSource(name string) views.Source { return views.KeptSource{State: stdio.State(name)} }
|
||||
|
||||
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
|
||||
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
|
||||
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, writer views.WireWriter, errs chan<- error, logf func(string, ...any)) {
|
||||
process.AwaitBucket(ctx, js, nc, marksBucket(), "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
|
||||
em := viewsEmitter(js, account, mk, writer, logf)
|
||||
for _, state := range []string{"asks", "differences", assign.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
|
||||
state := state
|
||||
start := func(kv jetstream.KeyValue) {
|
||||
if err := em.Start(ctx, state, kv, errs); err != nil {
|
||||
logf("%v: its readers ask its first view at each change until the assigner starts again", err)
|
||||
}
|
||||
}
|
||||
if kv := kvs[viewedStates()[state]]; kv != nil {
|
||||
start(kv)
|
||||
continue
|
||||
}
|
||||
process.AwaitBucket(ctx, js, nc, viewedStates()[state], "its changes are not said on the views family", logf).Ready(start)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
|
||||
// view-unsaid, in its findings, written through its finding writer (review round 2); with none it is said in the log only.
|
||||
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
|
||||
return views.Emitter{Owner: assignment.Module(), Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logf}
|
||||
}
|
||||
|
||||
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
|
||||
func firstViewsBucket() string { return assignment.Module() + "_" + views.StatusState }
|
||||
|
||||
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195, A-26): the
|
||||
// module's views reader adopts each branch's follower, so it asks the seat mesh-catalog's verb `revision` (the trunk's, then
|
||||
// each branch a registration names), follows its event `catalog-revision`, asks once more on a gap, and past the bound raises
|
||||
// the finding naming the seat and the verb, cleared by the answer, in the assigner's findings. Its status is read by
|
||||
// assigner_first_views and asked again by assigner_first_view_again.
|
||||
type catalogViews struct {
|
||||
js jetstream.JetStream
|
||||
account process.Account
|
||||
call func(string, any) (json.RawMessage, error)
|
||||
writer views.WireWriter
|
||||
status *process.AwaitedBucket
|
||||
logf func(string, ...any)
|
||||
hub *process.EventHub
|
||||
|
||||
h *halves
|
||||
// candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made;
|
||||
// before that a candidate's revision is said not taken.
|
||||
candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error]
|
||||
// answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers,
|
||||
// then its views family; nil when it is not followed.
|
||||
answers *views.Picture
|
||||
}
|
||||
|
||||
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
|
||||
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
|
||||
// seconds). A test shortens them.
|
||||
var firstViewBound, firstViewWait time.Duration
|
||||
|
||||
// start adopts the trunk's catalog half through the reader and starts the reader; it answers how many settles to wait for.
|
||||
func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
|
||||
reader := &views.ProcessReader{Module: assignment.Module(), Call: iv.call, Logf: iv.logf, Hub: iv.hub, Bound: firstViewBound, Wait: firstViewWait}
|
||||
if iv.writer != nil {
|
||||
reader.Findings = func() views.WireWriter { return iv.writer }
|
||||
}
|
||||
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
|
||||
if err := iv.h.follow(); err != nil {
|
||||
iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err)
|
||||
}
|
||||
// Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185).
|
||||
if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
|
||||
if f := iv.candidates.Load(); f != nil {
|
||||
return (*f)(ctx, ev)
|
||||
}
|
||||
return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision)
|
||||
}); err != nil {
|
||||
iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err)
|
||||
}
|
||||
// The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket.
|
||||
if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil {
|
||||
iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err)
|
||||
} else {
|
||||
iv.answers = pic
|
||||
}
|
||||
reader.Start(ctx, iv.js, iv.account, iv.status)
|
||||
return 1
|
||||
}
|
||||
|
||||
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
|
||||
// findings state it is said in the log alone.
|
||||
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
|
||||
w := heard.Writer{Observer: assignment.Module(), Logf: logf}
|
||||
if findings.Put != nil {
|
||||
w.Put = func(_ string, value []byte) error {
|
||||
var f mcseat.Finding
|
||||
if err := json.Unmarshal(value, &f); err != nil {
|
||||
return err
|
||||
}
|
||||
return findings.Put(f)
|
||||
}
|
||||
}
|
||||
if findings.Delete != nil {
|
||||
w.Delete = func(key string) error {
|
||||
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
|
||||
return nil // none stands
|
||||
}
|
||||
return findings.Delete(key)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// wantDropTool drops one want of a node for a module by hand (design 47 H §765, H-A-58): a held or failed want of the earlier
|
||||
// model goes at the clean state, its key deleted and the act written in the hand-act log; the next pass writes the want of the
|
||||
// new form.
|
||||
const wantDropTool = "assigner_want_drop"
|
||||
|
||||
// wantsState is the state the wants are kept in.
|
||||
const wantsState = "wants"
|
||||
|
||||
// dropWant deletes the want of module on node, the act first recorded through the seat mesh-hand-acts.
|
||||
func dropWant(st seatStore, ask func(string, any) (json.RawMessage, error), node, module, why, by string) (any, error) {
|
||||
switch {
|
||||
case node == "" || module == "":
|
||||
return nil, errors.New("name the node and the module whose want is dropped")
|
||||
case why == "":
|
||||
return nil, errors.New("a want dropped by hand says why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
|
||||
}
|
||||
key, err := mwc.WantKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, _, found, err := st.get(wantsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the assigner holds no want under %s", key)
|
||||
}
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the drop of %s is not recorded, so the want stays: %w", key, err)
|
||||
}
|
||||
if err := st.del(wantsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the want %s is not dropped: %w", key, err)
|
||||
}
|
||||
return map[string]any{"key": key, "dropped": json.RawMessage(raw), "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// The kinds the all form counts what it dropped by (design 47 H-A-123).
|
||||
const (
|
||||
dropHeld = "held"
|
||||
dropCurrent = "current"
|
||||
dropUndecodable = "undecodable"
|
||||
)
|
||||
|
||||
// dropAllWants deletes every want key of the state wants, held or not, and every value of an earlier form the decoder refuses
|
||||
// (the all form, design 47 H-A-121 (2), H-A-123): each act recorded through the seat mesh-hand-acts first, as the one-item form
|
||||
// does; it answers what it dropped, counted per kind. A drop not recorded stops it, saying what went before.
|
||||
func dropAllWants(st seatStore, ask func(string, any) (json.RawMessage, error), why, by string) (any, error) {
|
||||
switch {
|
||||
case why == "":
|
||||
return nil, errors.New("a want dropped by hand says why")
|
||||
case by == "" || by == stdio.Unproven:
|
||||
return nil, errors.New("a want dropped by hand names who drops it, and the runtime proved no caller")
|
||||
}
|
||||
all, err := st.keys(wantsState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
counted := map[string]int{dropHeld: 0, dropCurrent: 0, dropUndecodable: 0}
|
||||
dropped := map[string][]string{}
|
||||
answer := func() map[string]any {
|
||||
return map[string]any{"dropped": dropped, "counted": counted, "by": by}
|
||||
}
|
||||
for _, key := range all {
|
||||
if !strings.HasPrefix(key, "want.") {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := st.get(wantsState, key)
|
||||
if err != nil {
|
||||
return answer(), fmt.Errorf("the want %s cannot be read, so the drop stops there: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
kind := wantKind(key, raw)
|
||||
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: wantDropTool + " " + key, Why: why + " (" + by + ")", Cause: "want-drop"})
|
||||
if err != nil {
|
||||
return answer(), fmt.Errorf("the drop of %s is not recorded, so it and the wants after it stay: %w", key, err)
|
||||
}
|
||||
if err := st.del(wantsState, key); err != nil {
|
||||
return answer(), fmt.Errorf("the want %s is not dropped: %w", key, err)
|
||||
}
|
||||
counted[kind]++
|
||||
dropped[kind] = append(dropped[kind], key+" (hand act "+taken.Log+")")
|
||||
}
|
||||
return answer(), nil
|
||||
}
|
||||
|
||||
// wantKind says how the all form counts a want: undecodable when its key or value is of no form the decoder takes, held when it
|
||||
// holds, current otherwise (design 47 H-A-123).
|
||||
func wantKind(key string, raw []byte) string {
|
||||
if _, err := mwc.ParseWantNodeKey(key); err == nil {
|
||||
if _, err := mwc.DecodeNodeWant(raw); err != nil {
|
||||
return dropUndecodable
|
||||
}
|
||||
return dropCurrent
|
||||
}
|
||||
if _, _, err := mwc.ParseWantKey(key); err != nil {
|
||||
return dropUndecodable
|
||||
}
|
||||
w, err := mwc.DecodeWant(raw)
|
||||
switch {
|
||||
case err != nil:
|
||||
return dropUndecodable
|
||||
case w.Hold:
|
||||
return dropHeld
|
||||
}
|
||||
return dropCurrent
|
||||
}
|
||||
|
||||
// wantDropTools is the tool over the runtime's states.
|
||||
func wantDropTools(st seatStore, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: wantDropTool, Description: "Drop one want by hand (design 47 H §765): the want of module on node is " +
|
||||
"deleted from the state wants, as the clean state drops a held or failed want of the earlier model; the next pass writes the " +
|
||||
"want of the new form. Recorded as a hand act first; refused with no want under the key. With all true and no node or module, " +
|
||||
"every want key goes, held or not, and every value of an earlier form the decoder refuses, each recorded as a hand act; it " +
|
||||
"answers what it dropped, counted per kind (held, current, undecodable).",
|
||||
Input: schema(map[string]string{"node": "the node's name", "module": "the module's name", "why": "why the want is dropped",
|
||||
"all": "true drops every want of the state wants, naming no node or module"}, []string{"why"}, "all"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if arg(a, "all") == "true" {
|
||||
if arg(a, "node") != "" || arg(a, "module") != "" {
|
||||
return nil, errors.New("the all form drops every want: name no node or module with it")
|
||||
}
|
||||
return dropAllWants(st, ask, arg(a, "why"), stdio.CallerOf(a))
|
||||
}
|
||||
return dropWant(st, ask, arg(a, "node"), arg(a, "module"), arg(a, "why"), stdio.CallerOf(a))
|
||||
}}}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
)
|
||||
|
||||
// A12/AC1 (design 47 H §765, H-A-58): assigner_want_drop with node, module and why records the act in the hand-act log first
|
||||
// and then deletes that want; with no want under the key, no why or no proven caller, and when the act is not recorded, nothing is
|
||||
// dropped.
|
||||
func TestAWantIsDroppedByHandAndRecorded(t *testing.T) {
|
||||
key, _ := mwc.WantKey("ace", "web")
|
||||
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
|
||||
for _, c := range []struct {
|
||||
name, node, module, why, by string
|
||||
refuse error
|
||||
dropped bool
|
||||
}{
|
||||
{"dropped", "ace", "web", "the clean state", "person.jochen", nil, true},
|
||||
{"no want there", "ace", "db", "the clean state", "person.jochen", nil, false},
|
||||
{"no why", "ace", "web", "", "person.jochen", nil, false},
|
||||
{"no caller", "ace", "web", "the clean state", "", nil, false},
|
||||
{"not recorded", "ace", "web", "the clean state", "person.jochen", errors.New("no responders"), false},
|
||||
} {
|
||||
st := memStore{wantsState: &memKV{values: map[string][]byte{key: held}}}
|
||||
var asked []string
|
||||
ask := func(verb string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, verb+" "+string(raw))
|
||||
return json.RawMessage(`{}`), c.refuse
|
||||
}
|
||||
out, err := dropWant(st, ask, c.node, c.module, c.why, c.by)
|
||||
_, there := st[wantsState].values[key]
|
||||
switch {
|
||||
case c.dropped && (err != nil || there || len(asked) != 1 || !strings.Contains(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], wantDropTool+" "+key)):
|
||||
t.Errorf("%s: %v, %v; the want left %v, asked %v", c.name, out, err, there, asked)
|
||||
case !c.dropped && (err == nil || !there):
|
||||
t.Errorf("%s: %v, %v; the want left %v, want it refused and kept", c.name, out, err, there)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-121 (2), H-A-123: assigner_want_drop's all form drops every want key, held or not, and every value of an earlier form,
|
||||
// each recorded as a hand act first, answering the count per kind; a held fingerprint's key stays; a drop not recorded stops it;
|
||||
// the all form names no node or module.
|
||||
func TestEveryWantIsDroppedByHandAndCounted(t *testing.T) {
|
||||
heldKey, _ := mwc.WantKey("ace", "web")
|
||||
curKey, _ := mwc.WantKey("ace", "db")
|
||||
nodeKey, _ := mwc.WantNodeKey("ace")
|
||||
oldKey, _ := mwc.WantKey("g14", "web")
|
||||
fpKey, _ := mwc.HeldKey("web", fpTest)
|
||||
held, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Put: true, Hold: true, Words: "put: x"})
|
||||
cur, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest})
|
||||
node, _ := mwc.EncodeNodeWant(mwc.NodeWant{Self: true})
|
||||
values := func() map[string][]byte {
|
||||
return map[string][]byte{heldKey: held, curKey: cur, nodeKey: node, oldKey: []byte(`{"fingerprint":"old"}`), fpKey: []byte(`{}`)}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
refuse error
|
||||
left int
|
||||
}{
|
||||
{"dropped", nil, 1},
|
||||
{"not recorded", errors.New("no responders"), 5},
|
||||
} {
|
||||
st := memStore{wantsState: &memKV{values: values()}}
|
||||
var asked []string
|
||||
ask := func(verb string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, string(raw))
|
||||
return json.RawMessage(`{}`), c.refuse
|
||||
}
|
||||
out, err := dropAllWants(st, ask, "the clean state", "person.jochen")
|
||||
if left := len(st[wantsState].values); left != c.left || (c.refuse == nil) != (err == nil) {
|
||||
t.Errorf("%s: %v, %v; %d keys left, want %d", c.name, out, err, left, c.left)
|
||||
}
|
||||
if c.refuse != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := st[wantsState].values[fpKey]; !ok {
|
||||
t.Errorf("the held fingerprint's key %s is dropped: it is no want", fpKey)
|
||||
}
|
||||
counted := out.(map[string]any)["counted"].(map[string]int)
|
||||
if counted[dropHeld] != 1 || counted[dropCurrent] != 2 || counted[dropUndecodable] != 1 || len(asked) != 4 {
|
||||
t.Errorf("counted %v, asked %d hand acts; want held 1, current 2, undecodable 1 and 4 acts", counted, len(asked))
|
||||
}
|
||||
}
|
||||
if _, err := dropAllWants(memStore{wantsState: &memKV{values: values()}}, nil, "", "person.jochen"); err == nil {
|
||||
t.Error("the all form dropped with no why")
|
||||
}
|
||||
run := wantDropTools(memStore{wantsState: &memKV{values: values()}}, nil)[0].Run
|
||||
if _, err := run(map[string]any{"all": "true", "node": "ace", "why": "x"}); err == nil || !strings.Contains(err.Error(), "name no node") {
|
||||
t.Errorf("the all form with a node: %v, want it refused", err)
|
||||
}
|
||||
}
|
||||
@@ -4,35 +4,31 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-module-assigner's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names beside every composition each node published (ADR 0593 §8, composed.go); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
|
||||
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = buckets.PoliciesState
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
composedState = "mesh-controller.composed"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
@@ -45,59 +41,52 @@ type wantsTools struct {
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
emit func(event, id string, body any) error
|
||||
now func() time.Time
|
||||
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
|
||||
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
|
||||
// a large state (issue 1953 L5f). Nil reads key by key.
|
||||
watched *stdio.WatchedStates
|
||||
// revisions is the kept data assignment-revisions the verbs revision, entry and difference read (design 47 lines
|
||||
// 1365-1366); nil serves them not.
|
||||
revisions assigngraph.Objects
|
||||
}
|
||||
|
||||
// wantsEventSubject is the subject of one of the seat's events.
|
||||
func wantsEventSubject(event string) string {
|
||||
return "mesh.seat." + meshwants.Seat + ".event." + event
|
||||
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
|
||||
}
|
||||
|
||||
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
return s.watched.Values(state)
|
||||
}
|
||||
return s.direct(state)
|
||||
// leased says the controller's lease is handed away.
|
||||
func (s *wantsTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// direct reads a state whole, in pages where the runtime has them (readAll).
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
all, err := readAll(s.st, state)
|
||||
// values reads every key of a state.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
out := make(map[string][]byte, len(all))
|
||||
for k, v := range all {
|
||||
out[k] = v.raw
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifests is every module the indexer holds, with its trunk manifest and fingerprint: both asked of the indexer's first
|
||||
// views of modules and manifests (indexerView), never read from its bus state.
|
||||
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
mods, err := indexerView(s.ask, indexerModules)
|
||||
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
|
||||
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
|
||||
mods, err := s.values(modulesState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
kept, err := indexerView(s.ask, indexerManifests)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out, held := map[string]catalogue.Manifest{}, map[string]meshcatalog.ModulesByModuleValue{}
|
||||
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
|
||||
for name, raw := range mods {
|
||||
m, err := meshcatalogclient.DecodeModule(raw)
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -105,16 +94,15 @@ func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]mesh
|
||||
if m.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
mraw, found := kept[m.Manifest]
|
||||
if !found {
|
||||
mraw, _, found, err := s.st.get(manifestsState, m.Manifest)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
stored, err := meshcatalogclient.DecodeManifest(mraw)
|
||||
kept, err := delivery.DecodeManifest(mraw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
|
||||
if man, err := catalogue.Decode([]byte(stored.Manifest)); err == nil {
|
||||
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil {
|
||||
out[name] = man
|
||||
}
|
||||
}
|
||||
@@ -131,39 +119,29 @@ type policyRow struct {
|
||||
}
|
||||
|
||||
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
|
||||
// back with default; with compare each node's composition and the assigner's findings.
|
||||
// back with default; with compare list the splits.
|
||||
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
manifests, _, err := s.manifests()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
|
||||
raw, err := s.values(policiesState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
haves, err := s.values(haveState)
|
||||
composed, err := s.values(composedState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
|
||||
if splits == nil {
|
||||
splits = []assign.Split{}
|
||||
}
|
||||
out := map[string]any{"composed": composed}
|
||||
// And what every node published (ADR 0593 §8), read only to compare.
|
||||
if values, err := s.values(buckets.ComposedRead); err != nil {
|
||||
out["published-unread"] = err.Error()
|
||||
} else {
|
||||
out["published"] = published(values)
|
||||
}
|
||||
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
|
||||
if found, err := s.values(assign.FindingsState); err != nil {
|
||||
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
|
||||
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
} else {
|
||||
keys := []string{}
|
||||
@@ -195,14 +173,18 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
|
||||
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
|
||||
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
|
||||
// and says policy-changed. The bus is never rolled: a choice for it is refused.
|
||||
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("a policy is set for one module: name it")
|
||||
}
|
||||
if _, ok := manifests[module]; !ok {
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
if catalogue.ProvidesBus(m) {
|
||||
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("a person's policy says why")
|
||||
}
|
||||
@@ -224,8 +206,8 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, rerr := rec.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
recorded = "not recorded: " + rerr.Error()
|
||||
@@ -264,37 +246,37 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
|
||||
if bus == "" {
|
||||
return nil, errors.New("the indexer holds no module that provides the bus")
|
||||
}
|
||||
newest := held[bus].Fp2
|
||||
newest := held[bus].FP2
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nodes []busNode
|
||||
for key, raw := range haves {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode || module != bus {
|
||||
continue
|
||||
}
|
||||
h, err := nodeengineclient.DecodeHave(raw)
|
||||
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
|
||||
h, err := delivery.DecodeHave(raw)
|
||||
if err != nil || h.Phase == delivery.PhaseRemoved {
|
||||
continue
|
||||
}
|
||||
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
|
||||
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
|
||||
}
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
|
||||
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
|
||||
if steps, err := s.ask("seat:"+meshhandacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
|
||||
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
|
||||
out["steps-unread"] = err.Error()
|
||||
} else {
|
||||
out["steps"] = steps
|
||||
}
|
||||
kept, err := s.values(putsState)
|
||||
puts, err := s.values(putsState)
|
||||
if err != nil {
|
||||
out["puts-unread"] = err.Error()
|
||||
} else {
|
||||
mine := map[string]json.RawMessage{}
|
||||
for k, v := range kept {
|
||||
if _, module, err := puts.ParseKey(k); err == nil && module == bus {
|
||||
for k, v := range puts {
|
||||
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
|
||||
mine[k] = v
|
||||
}
|
||||
}
|
||||
@@ -303,24 +285,16 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>; the tool runner describes them with the
|
||||
// seat's definition from the seat list (ADR 0635 §1), so the tools carry none.
|
||||
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
|
||||
func (s *wantsTools) tools() []stdio.Tool {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
|
||||
run := map[string]func(map[string]any) (any, error){
|
||||
meshwants.VerbUpgrade: func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
|
||||
meshwants.VerbBus: s.doBus,
|
||||
}
|
||||
if s.revisions != nil {
|
||||
for v, f := range graphVerbs(s.revisions) {
|
||||
run[v] = f
|
||||
}
|
||||
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
|
||||
"bus": s.doBus,
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range meshwants.Verbs() {
|
||||
if run[v] == nil {
|
||||
continue // a verb of the seat this module does not serve yet: revision, entry, difference (AS11)
|
||||
}
|
||||
out = append(out, stdio.Tool{Name: meshwants.Seat + "." + v, Run: run[v]})
|
||||
for _, v := range seat.Serves {
|
||||
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// Why a want is what it is (lead-47 2026-11-15, the tools rule: novox's want of the indexer stood unwritten for a day and nothing
|
||||
// said why). Each pass writes, in the state written beside the pipeline ends, one key `why.<node>.<module>`: the step of the pass
|
||||
// that decided the want and what it read there. A key whose time stands still says the pass no longer judges that module.
|
||||
// assigner_why reads them.
|
||||
|
||||
// whyPrefix starts every key of a want's why in the state written; a pipeline end's key starts with a module's name.
|
||||
const whyPrefix = "why."
|
||||
|
||||
// whyTool is the tool that reads them.
|
||||
const whyTool = "assigner_why"
|
||||
|
||||
// The steps of the pass that decide a want.
|
||||
const (
|
||||
whyNoPipeline = "no-passed-pipeline" // no version of the module released on the branch passed its pipeline: the want written stays
|
||||
whyPassed = "passed-pipeline" // the want names the newest version released on the branch whose pipeline passed (design 47 H §750)
|
||||
)
|
||||
|
||||
// whyStep is the step that decided a want: a passed pipeline's version, or none yet.
|
||||
func whyStep(passed bool) string {
|
||||
if passed {
|
||||
return whyPassed
|
||||
}
|
||||
return whyNoPipeline
|
||||
}
|
||||
|
||||
// wantWhy is one key's value.
|
||||
type wantWhy struct {
|
||||
At time.Time `json:"at"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// Fp, Version and Pipeline are the build the want names and the pipeline whose passed end released it (design 47 H §750).
|
||||
Fp string `json:"fp,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Pipeline string `json:"pipeline,omitempty"`
|
||||
Step string `json:"step"`
|
||||
// Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before).
|
||||
Before *mwc.Want `json:"before,omitempty"`
|
||||
After *mwc.Want `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
func whyKey(node, module string) string { return whyPrefix + node + "." + module }
|
||||
|
||||
// sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear.
|
||||
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
one := w
|
||||
one.At = time.Now().UTC()
|
||||
if b, ok := before[node]; ok {
|
||||
one.Before = &b
|
||||
}
|
||||
if out != nil {
|
||||
if a, ok := out.Wants[node]; ok {
|
||||
one.After = &a
|
||||
}
|
||||
}
|
||||
same := one
|
||||
same.At = time.Time{}
|
||||
cmp, _ := json.Marshal(same)
|
||||
key := whyKey(node, module)
|
||||
if p.whys[key] == string(cmp) {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(one)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, raw); err != nil {
|
||||
p.logf("the why of %s on %s is not written: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
p.whys[key] = string(cmp)
|
||||
}
|
||||
}
|
||||
|
||||
// whyTools is the tool assigner_why: every want's why, or those of one module or one node.
|
||||
func whyTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline yet, or a passed pipeline's version), the build, version and pipeline it names, the want before and the one written. A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
|
||||
Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
module, node := arg(args, "module"), arg(args, "node")
|
||||
sort.Strings(ks)
|
||||
out := map[string]wantWhy{}
|
||||
for _, k := range ks {
|
||||
rest, ok := strings.CutPrefix(k, whyPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
n, mod, _ := strings.Cut(rest, ".")
|
||||
if (module != "" && mod != module) || (node != "" && n != node) {
|
||||
continue
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
var w wantWhy
|
||||
if json.Unmarshal(raw, &w) == nil {
|
||||
out[rest] = w
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
|
||||
// sayPass writes, under `why.pass`, whether the last pass judged the wants at all, once per change of its words.
|
||||
func (p *passer) sayPass(ctx context.Context, words string) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
if p.whys[whyPrefix+"pass"] == words {
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(wantWhy{At: time.Now().UTC(), Step: words})
|
||||
if _, err := kv.Put(ctx, whyPrefix+"pass", raw); err == nil {
|
||||
p.whys[whyPrefix+"pass"] = words
|
||||
}
|
||||
}
|
||||
@@ -1,31 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
)
|
||||
|
||||
// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -38,29 +52,11 @@ type world struct {
|
||||
changes map[string]uint64
|
||||
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
|
||||
meta map[string]entryMeta
|
||||
// ends are the done builds' ends the builder says (build.ended), by module and fingerprint (buildlog.go).
|
||||
ends builtEnds
|
||||
// released are the catalog's version-released heard, by module and source revision (buildended.go).
|
||||
released releasedVersions
|
||||
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
|
||||
edges map[string]map[string][]meshcatalog.Edge
|
||||
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
|
||||
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
assignmentKeys *mmac.Keys
|
||||
// first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the
|
||||
// branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532),
|
||||
// and never fixed in code (§7).
|
||||
first string
|
||||
// presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389).
|
||||
presence map[string]presenceSaid
|
||||
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
|
||||
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
|
||||
running map[string]nodeengine.Running
|
||||
runningHeard bool
|
||||
// modules and graphs are, by watched branch and module, the catalog half the assigner follows per branch (design 47 G §14,
|
||||
// §184, §195), as the value and the graph node the rules read: written by putModule from the entries of meshcatalogclient.Catalog.
|
||||
modules map[string]map[string]meshcatalog.ModulesByModuleValue
|
||||
graphs map[string]map[string]graph.Node
|
||||
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
|
||||
trunkBuilt map[string]delivery.BuildLogEntry
|
||||
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
|
||||
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
placement *delivery.PlacementKeys
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
@@ -78,9 +74,7 @@ type parsed struct {
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(),
|
||||
running: map[string]nodeengine.Running{},
|
||||
modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}}
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
@@ -93,8 +87,8 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
|
||||
return
|
||||
}
|
||||
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
|
||||
if bucket == plc.AssignmentsBucket() {
|
||||
_, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,25 +109,42 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
|
||||
if value == nil {
|
||||
delete(w.values[bucket], key)
|
||||
if bucket == plc.AssignmentsBucket() {
|
||||
_, _ = w.assignmentKeys.Put(key, nil, 0, true)
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, nil, 0, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == plc.AssignmentsBucket() {
|
||||
if bucket == delivery.PlacementBucket {
|
||||
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
|
||||
_, _ = w.assignmentKeys.Put(key, value, 0, false)
|
||||
_, _ = w.placement.Put(key, value, 0, false)
|
||||
}
|
||||
if bucket == buckets.Have {
|
||||
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
|
||||
w.healthy[key] = append(list, h.Fp)
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
w.healthy[key] = append(list, h.FP)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
|
||||
func (w *world) lastHealthy(node, module, fp string) string {
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
list := w.healthy[key]
|
||||
for i := len(list) - 1; i >= 0; i-- {
|
||||
if list[i] != fp {
|
||||
return list[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// read is a copy of what a bucket holds now.
|
||||
func (w *world) read(bucket string) map[string][]byte {
|
||||
w.mu.Lock()
|
||||
@@ -145,46 +156,111 @@ func (w *world) read(bucket string) map[string][]byte {
|
||||
return out
|
||||
}
|
||||
|
||||
// getter reads one key's value; errNotThere when the key does not exist.
|
||||
type getter func(ctx context.Context, key string) ([]byte, error)
|
||||
|
||||
var errNotThere = errors.New("no such key")
|
||||
|
||||
// manifest is the manifest of a digest the catalog half said (putModule keeps each once); one not said yet is not readable.
|
||||
func (w *world) manifest(digest string) parsed {
|
||||
// kvGetter reads keys of a bucket.
|
||||
func kvGetter(kv jetstream.KeyValue) getter {
|
||||
return func(ctx context.Context, key string) ([]byte, error) {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, errNotThere
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
|
||||
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if p, ok := w.manifests[digest]; ok {
|
||||
p, ok := w.manifests[digest]
|
||||
w.mu.Unlock()
|
||||
if ok {
|
||||
return p
|
||||
}
|
||||
return parsed{err: "the catalog half holds no manifest of digest " + digest + " yet"}
|
||||
}
|
||||
|
||||
// assigned is each node's modules the seat mesh-module-assigner's assignments names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// assignments is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return parsed{err: err.Error()}
|
||||
}
|
||||
return out
|
||||
value, err := get(ctx, key)
|
||||
if errors.Is(err, errNotThere) {
|
||||
return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"}
|
||||
}
|
||||
if err != nil {
|
||||
return parsed{err: err.Error()}
|
||||
}
|
||||
m, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
p = parsed{err: err.Error()}
|
||||
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
|
||||
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
|
||||
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
|
||||
p = parsed{module: m.Module, err: err.Error()}
|
||||
} else {
|
||||
p = parsed{module: m.Module, manifest: man}
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.manifests[digest] = p
|
||||
w.mu.Unlock()
|
||||
return p
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's assignments (ADR 0567 §1), and whether any assignments key was read: a key
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
|
||||
// the lease write's own check refuses that order (ADR 0547 §11).
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, all := w.assignedByController()
|
||||
if away := w.handedAway(); away && seeded {
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, away)
|
||||
return both, map[string]assignment{}
|
||||
}
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, false)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
|
||||
// the controller's lease is handed away, since the placement is then the one source.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, _ := w.assignedByController()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
|
||||
return differ
|
||||
}
|
||||
|
||||
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
|
||||
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
|
||||
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
|
||||
func (w *world) handedAway() bool {
|
||||
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
|
||||
return away
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
// that does not read places nothing on its node and is said by the indexer's own read tool.
|
||||
func (w *world) placed() (map[string][]string, bool) {
|
||||
out := map[string][]string{}
|
||||
for node, p := range w.assignments() {
|
||||
out[node] = mmac.Assigned(p)
|
||||
for node, p := range w.placements() {
|
||||
out[node] = p.Assigned()
|
||||
}
|
||||
return out, len(w.read(plc.AssignmentsBucket())) > 0
|
||||
return out, len(w.read(delivery.PlacementBucket)) > 0
|
||||
}
|
||||
|
||||
// assignments is every node's assignments, assembled from its keys in either form (mmac.Keys).
|
||||
func (w *world) assignments() map[string]mma.Assignments {
|
||||
out, _ := w.assignmentKeys.AllRevisions()
|
||||
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
|
||||
func (w *world) placements() map[string]delivery.Placement {
|
||||
out, _ := w.placement.AllRevisions()
|
||||
return out
|
||||
}
|
||||
|
||||
// placedNodes is every node the seat's assignments names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
|
||||
// so their wants name the build alone and the seventh rule does not hold them.
|
||||
func (w *world) placedNodes() map[string]bool {
|
||||
placed, _ := w.placed()
|
||||
@@ -195,187 +271,253 @@ func (w *world) placedNodes() map[string]bool {
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkOf is every module the catalog half of branch holds, with its value (design 47 G §184).
|
||||
func (w *world) trunkOf(branch string) map[string]meshcatalog.ModulesByModuleValue {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return maps.Clone(w.modules[branch])
|
||||
}
|
||||
|
||||
// graphOf is the module graph of the catalog half of branch.
|
||||
func (w *world) graphOf(branch string) graph.Graph {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return graph.Graph(maps.Clone(w.graphs[branch]))
|
||||
}
|
||||
|
||||
// branches are the watched branches the catalog half is held of, the trunk's first.
|
||||
func (w *world) branches() []string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
out := slices.Sorted(maps.Keys(w.modules))
|
||||
if i := slices.Index(out, w.first); i > 0 {
|
||||
out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first)
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
}
|
||||
return out
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
}
|
||||
return out, all
|
||||
}
|
||||
|
||||
// trunk is every module any branch's catalog half holds, each with its value on the first branch that holds it (the trunk's
|
||||
// first): the modules that exist, for the parts that ask which, not at which fingerprint a node runs it.
|
||||
func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue {
|
||||
out := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
for _, b := range w.branches() {
|
||||
for name, v := range w.trunkOf(b) {
|
||||
if _, ok := out[name]; !ok {
|
||||
out[name] = v
|
||||
}
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
func (w *world) trunk() map[string]delivery.Module {
|
||||
out := map[string]delivery.Module{}
|
||||
for key, value := range w.read(delivery.ModulesBucket) {
|
||||
name, err := delivery.ParseModulesKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if m, err := delivery.DecodeModule(value); err == nil {
|
||||
out[name] = m
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// putModule takes one module of branch's catalog half in: its value, its graph node and its parsed manifest by the digest the
|
||||
// value names.
|
||||
func (w *world) putModule(branch, module string, v meshcatalog.ModulesByModuleValue, n graph.Node, m *parsed) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.modules[branch] == nil {
|
||||
w.modules[branch], w.graphs[branch] = map[string]meshcatalog.ModulesByModuleValue{}, map[string]graph.Node{}
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned, _ = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
m.Digests[name] = mod.Manifest
|
||||
p := w.manifest(ctx, manifests, mod.Manifest)
|
||||
if p.err != "" {
|
||||
m.Unreadable[name] = p.err
|
||||
continue
|
||||
}
|
||||
m.Manifests[name] = p.manifest
|
||||
}
|
||||
w.modules[branch][module], w.graphs[branch][module] = v, n
|
||||
if m != nil && v.Manifest != "" {
|
||||
w.manifests[v.Manifest] = *m
|
||||
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
|
||||
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
|
||||
// assignments alone are composed, as before any placement.
|
||||
if _, seeded := w.placed(); seeded {
|
||||
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
|
||||
if err != nil {
|
||||
m.InputUnread = err.Error()
|
||||
} else {
|
||||
m.Input = &in
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// dropModule takes a module out of branch's catalog half.
|
||||
func (w *world) dropModule(branch, module string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
delete(w.modules[branch], module)
|
||||
delete(w.graphs[branch], module)
|
||||
delete(w.edges[branch], module)
|
||||
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
|
||||
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
|
||||
// composition it asks resolves, and stands in for what it does not carry).
|
||||
type worldReader struct {
|
||||
w *world
|
||||
manifests getter
|
||||
}
|
||||
|
||||
// putEdges keeps the edges into module's artifacts that branch's catalog entry names.
|
||||
func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.edges == nil {
|
||||
w.edges = map[string]map[string][]meshcatalog.Edge{}
|
||||
}
|
||||
if w.edges[branch] == nil {
|
||||
w.edges[branch] = map[string][]meshcatalog.Edge{}
|
||||
}
|
||||
w.edges[branch][module] = edges
|
||||
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
return r.w.placements(), nil
|
||||
}
|
||||
|
||||
// edgesOf is the edges into module's artifacts in branch's catalog half.
|
||||
func (w *world) edgesOf(branch, module string) []meshcatalog.Edge {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.edges[branch][module]
|
||||
}
|
||||
|
||||
// holdsBranch says whether the catalog half of branch has been taken in (even empty).
|
||||
func (w *world) holdsBranch(branch string) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.modules[branch] != nil
|
||||
}
|
||||
|
||||
// records is each node's record in its own have key (`<node>.node`): the nodes an assignments proposal is answered by,
|
||||
// with those assignments names (ADR 0593 §8).
|
||||
func (w *world) records() map[string]nodeengine.NodeRecord {
|
||||
out := map[string]nodeengine.NodeRecord{}
|
||||
for key, value := range w.read(buckets.Have) {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
for key, value := range r.w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389).
|
||||
type presenceSaid struct {
|
||||
away bool
|
||||
at time.Time
|
||||
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.w.trunk(), nil
|
||||
}
|
||||
|
||||
// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two
|
||||
// streams, read whole in either order, leave each machine as its last event says.
|
||||
func (w *world) heardPresence(machine string, away bool, at time.Time) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.presence == nil {
|
||||
w.presence = map[string]presenceSaid{}
|
||||
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if was, ok := w.presence[machine]; ok && at.Before(was.at) {
|
||||
return
|
||||
raw, err := r.manifests(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.presence[machine] = presenceSaid{away: away, at: at}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
// awayNow is every machine whose last presence event heard is machine-away.
|
||||
func (w *world) awayNow() map[string]bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
out := map[string]bool{}
|
||||
for m, p := range w.presence {
|
||||
if p.away {
|
||||
out[m] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to
|
||||
// the seat's verb is heard (novox/hq ADR 0668 §3).
|
||||
func (w *world) seatHeard(seat string) bool {
|
||||
online := w.online()
|
||||
for _, value := range w.read(plc.SeatHoldersBucket()) {
|
||||
var sh mma.SeatHolders
|
||||
if json.Unmarshal(value, &sh) != nil || sh.Seat != seat {
|
||||
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
|
||||
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
|
||||
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
var out map[string]any
|
||||
prefix := node + "." + module + "."
|
||||
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
|
||||
name, ok := strings.CutPrefix(key, prefix)
|
||||
if !ok || strings.Contains(name, ".") {
|
||||
continue
|
||||
}
|
||||
for _, h := range sh.Holders {
|
||||
if online[h.Node] {
|
||||
return true
|
||||
if v, laid := catalogue.LaidValue(raw); laid {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[name] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
|
||||
// composes from the manifests the indexer read, and reads no build and no archive.
|
||||
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
|
||||
}
|
||||
|
||||
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
all, err := r.Values(ctx, node, module)
|
||||
var out map[string]any
|
||||
for _, k := range keys {
|
||||
if v, ok := all[k]; ok {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
|
||||
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
|
||||
heads := map[string]string{}
|
||||
for _, p := range c.Pulls {
|
||||
heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head
|
||||
}
|
||||
ch.Held = assign.HeldOf(w.read(keptBucket), heads)
|
||||
definedIn := map[string]map[string]bool{}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository == assign.SDKRepository {
|
||||
ch.SDKHead = p.Head.Head
|
||||
}
|
||||
ch.Members[p.Repository] = p.Head.Head
|
||||
for _, name := range p.Modules {
|
||||
fp, ok := p.Fps[name]
|
||||
if !ok || fp.Manifest == "" {
|
||||
if ok && fp.None2 == noModuleWords {
|
||||
ch.Removed = append(ch.Removed, name)
|
||||
continue
|
||||
}
|
||||
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
|
||||
}
|
||||
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
|
||||
ch.Rebuilt++
|
||||
}
|
||||
got := w.manifest(ctx, manifests, fp.Manifest)
|
||||
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
|
||||
return ch, got.err
|
||||
}
|
||||
ch.Digests[name] = fp.Manifest
|
||||
if got.err != "" {
|
||||
ch.Unreadable[name] = got.err
|
||||
continue
|
||||
}
|
||||
ch.Manifests[got.module] = got.manifest
|
||||
if definedIn[got.module] == nil {
|
||||
definedIn[got.module] = map[string]bool{}
|
||||
}
|
||||
definedIn[got.module][p.Repository] = true
|
||||
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
|
||||
definedIn[got.module][t.Repository] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
for name, repos := range definedIn {
|
||||
if len(repos) > 1 {
|
||||
for r := range repos {
|
||||
ch.Twice[name] = append(ch.Twice[name], r)
|
||||
}
|
||||
sort.Strings(ch.Twice[name])
|
||||
}
|
||||
}
|
||||
return ch, ""
|
||||
}
|
||||
|
||||
// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its
|
||||
// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2).
|
||||
// online is, by node, whether the watchman holds the node online.
|
||||
func (w *world) online() map[string]bool {
|
||||
away := w.awayNow()
|
||||
out := map[string]bool{}
|
||||
for _, node := range w.knownNodes() {
|
||||
out[node] = !away[node]
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
out[node] = p.Online
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// wantsOf is the wants the assigner wrote, by module then node.
|
||||
func (w *world) wantsOf() map[string]map[string]mwc.Want {
|
||||
out := map[string]map[string]mwc.Want{}
|
||||
for key, value := range w.read(buckets.Wants()) {
|
||||
node, module, err := mwc.ParseWantKey(key)
|
||||
func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||
out := map[string]map[string]delivery.Want{}
|
||||
for key, value := range w.read(delivery.AssignerWantsBucket) {
|
||||
node, module, err := delivery.ParseWantKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
want, err := mwc.DecodeWant(value)
|
||||
want, err := delivery.DecodeWant(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]mwc.Want{}
|
||||
out[module] = map[string]delivery.Want{}
|
||||
}
|
||||
out[module][node] = want
|
||||
}
|
||||
@@ -384,20 +526,24 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
|
||||
|
||||
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
|
||||
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
|
||||
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
|
||||
Reads: map[string][]string{}}
|
||||
n.Online = w.online()
|
||||
for key, value := range w.read(buckets.Have) {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
if err != nil {
|
||||
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
continue
|
||||
}
|
||||
if isNode { // what the node's engine reads is the state engine-reads' alone (design 47 H-A-104)
|
||||
if p, err := delivery.DecodePresence(value); err == nil {
|
||||
n.Online[node] = p.Online
|
||||
}
|
||||
}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHave(value); err == nil {
|
||||
if h, err := delivery.DecodeHave(value); err == nil {
|
||||
if n.Haves[node] == nil {
|
||||
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
n.Haves[node] = map[string]delivery.Have{}
|
||||
}
|
||||
n.Haves[node][module] = h
|
||||
}
|
||||
@@ -426,9 +572,15 @@ func (w *world) value(bucket, key string) []byte {
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
|
||||
// manifest but the controller's reads the controller's grants.
|
||||
func (w *world) grantsBucket() string { return buckets.Grants }
|
||||
// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's
|
||||
// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other
|
||||
// case, a lease not held here included.
|
||||
func (w *world) grantsBucket() string {
|
||||
raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil)
|
||||
bucket, _ := delivery.GrantSource(away)
|
||||
return bucket
|
||||
}
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
@@ -438,42 +590,76 @@ func (w *world) has(bucket, key string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
|
||||
func (w *world) knownNodes() []string {
|
||||
set := map[string]bool{}
|
||||
for node := range w.placedNodes() {
|
||||
set[node] = true
|
||||
}
|
||||
for node := range w.records() {
|
||||
set[node] = true
|
||||
// built says whether the builder's build of a module at a fingerprint is done.
|
||||
func (w *world) built(module, fp string) bool {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
w.mu.Lock()
|
||||
for node := range w.presence {
|
||||
set[node] = true
|
||||
}
|
||||
value, ok := w.values[delivery.BuilderBuildsBucket][key]
|
||||
w.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(set))
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(value)
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
|
||||
if !w.placedNodes()[node] {
|
||||
return assign.ComposedDigest{}, false
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
//
|
||||
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
|
||||
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
|
||||
// no want (and the pass says so, unplacedFindings).
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
if w.handedAway() {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
if w.handedAway() {
|
||||
return out // the controller composes no node from the lease write (ADR 0571 §6)
|
||||
}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
func (w *world) answer(key string) (mdc.Answer, bool) {
|
||||
func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[buckets.Answers][key]
|
||||
value, ok := w.values[delivery.AnswersBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return mdc.Answer{}, false
|
||||
return delivery.Answer{}, false
|
||||
}
|
||||
a, err := mdc.DecodeAnswer(value)
|
||||
a, err := delivery.DecodeAnswer(value)
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
@@ -492,32 +678,69 @@ func (w *world) changed(bucket, key string) uint64 {
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch).
|
||||
type logWatch func(handle func(stdio.LogEntry) error) error
|
||||
|
||||
// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the
|
||||
// module's last, becomes the module's running commit. It answers whether the commit changed.
|
||||
func (w *world) putBuilt(raw []byte) bool {
|
||||
e, err := delivery.DecodeBuildLogEntry(raw)
|
||||
if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 {
|
||||
return false
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
have, ok := w.trunkBuilt[e.Module]
|
||||
if ok && !e.At.After(have.At) {
|
||||
return false
|
||||
}
|
||||
w.trunkBuilt[e.Module] = e
|
||||
return !ok || have.Commit != e.Commit
|
||||
}
|
||||
|
||||
// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a
|
||||
// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build.
|
||||
func (w *world) runningCommits(module string) []string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if e, ok := w.trunkBuilt[module]; ok {
|
||||
return []string{e.Commit}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5).
|
||||
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
|
||||
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
|
||||
key, err := nodeengineclient.HaveNodeKey(node)
|
||||
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return assign.ComposedDigest{}, false
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
raw, ok := w.read(buckets.Have)[key]
|
||||
raw, ok := w.read(delivery.HaveBucket)[key]
|
||||
if !ok {
|
||||
return assign.ComposedDigest{}, false
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
h, err := nodeengineclient.DecodeHaveNode(raw)
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil || h.Composed == "" {
|
||||
return assign.ComposedDigest{}, false
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
|
||||
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are the nodes whose engine has a have and that no assignments names: they get no
|
||||
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
if !w.handedAway() {
|
||||
return nil
|
||||
}
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(buckets.Have) {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err == nil && isNode && !placed[node] {
|
||||
out = append(out, node)
|
||||
}
|
||||
@@ -525,46 +748,3 @@ func (w *world) unplacedNodes() []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted.
|
||||
func (w *world) trunkName() string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.first
|
||||
}
|
||||
|
||||
// setTrunk says which watched branch is the first.
|
||||
func (w *world) setTrunk(branch string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.first = branch
|
||||
}
|
||||
|
||||
// placedBranches is, by module and node, the watched branch each placement names ("" when it names none).
|
||||
func (w *world) placedBranches() map[string]map[string]string {
|
||||
out := map[string]map[string]string{}
|
||||
for node, a := range w.assignments() {
|
||||
for _, m := range a.Modules {
|
||||
if out[m.Module] == nil {
|
||||
out[m.Module] = map[string]string{}
|
||||
}
|
||||
out[m.Module][node] = m.Branch
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedOn is the modules the assignments place on node.
|
||||
func (w *world) assignedOn(node string) []string {
|
||||
var out []string
|
||||
for _, m := range w.assignments()[node].Modules {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignmentOf is node's assignments, or false where none is held.
|
||||
func (w *world) assignmentOf(node string) (mma.Assignments, bool) {
|
||||
a, ok := w.assignments()[node]
|
||||
return a, ok
|
||||
}
|
||||
|
||||
@@ -2,17 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// fakeManifests is the indexer's manifests bucket in memory.
|
||||
@@ -20,8 +19,8 @@ type fakeManifests map[string][]byte
|
||||
|
||||
func (f fakeManifests) add(t *testing.T, module, body string) string {
|
||||
t.Helper()
|
||||
digest := meshcatalogclient.ManifestDigest([]byte(body))
|
||||
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
|
||||
digest := delivery.ManifestDigest([]byte(body))
|
||||
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -36,79 +35,132 @@ func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
|
||||
return nil, errNotThere
|
||||
}
|
||||
|
||||
func fp(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
|
||||
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
|
||||
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
|
||||
for m := range modules {
|
||||
p.Modules = append(p.Modules, m)
|
||||
}
|
||||
slices.Sort(p.Modules)
|
||||
return p
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
|
||||
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||
// another repository as defined twice.
|
||||
func TestTheChangeOfACandidate(t *testing.T) {
|
||||
theWorldCountsTheValuesAKeyTakes(t)
|
||||
f := fakeManifests{}
|
||||
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
|
||||
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
|
||||
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
|
||||
pull(assign.SDKRepository, 3, nil),
|
||||
}}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
|
||||
if waits != "" {
|
||||
t.Fatalf("the change waits: %s", waits)
|
||||
}
|
||||
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
|
||||
t.Errorf("change = %+v", ch)
|
||||
}
|
||||
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
|
||||
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
|
||||
// saying for what; an unreadable manifest is the change's, not a wait.
|
||||
func TestAChangeWaitsForItsManifests(t *testing.T) {
|
||||
f := fakeManifests{}
|
||||
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
|
||||
for _, c := range []delivery.CandidatePull{
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
|
||||
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
|
||||
} {
|
||||
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
|
||||
if waits == "" {
|
||||
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
|
||||
}
|
||||
}
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["web"] == "" {
|
||||
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest of the change whose config value falls back to a predefined key (novox/hq ADR 0348 §7) is read, not
|
||||
// unreadable for the catalogue's predefined.json the assigner never holds: the merge gate judges the fallback in the
|
||||
// catalogue, and the indexer reads it the same way (catalogue.ParseManifestForInputs, issue 1751). A group whose
|
||||
// mesh-catalog member touched pacman and xterm failed its composition on this (group check 18df81380db3). The manifest is
|
||||
// xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"predefined"`) {
|
||||
t.Fatal("the fixture names no predefined key")
|
||||
}
|
||||
f := fakeManifests{}
|
||||
xterm := f.add(t, "xterm", string(raw))
|
||||
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||
Pulls: []delivery.CandidatePull{pull("novox/mesh-catalog", 1, map[string]delivery.CandidateModule{"xterm": {FP2: fp('b'), Manifest: xterm}})}}, nil)
|
||||
if waits != "" || ch.Unreadable["xterm"] != "" || ch.Manifests["xterm"].Module != "xterm" {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
|
||||
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
|
||||
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
|
||||
t.Errorf("changed = %d, want 2", got)
|
||||
}
|
||||
if got := w.changed(buckets.Have, "gone"); got != 0 {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
|
||||
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := grantsstate.PartGrantKey("web", fp)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{Accounts: []string{"ace.web"}})
|
||||
// Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant
|
||||
// only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it
|
||||
// holds it, a grant only the controller holds being the one that counts instead.
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
inPart bool
|
||||
granted bool
|
||||
name, lease string
|
||||
inPart, inController bool
|
||||
bucket string
|
||||
granted bool
|
||||
}{
|
||||
{"the part's grant", true, true},
|
||||
{"no grant", false, false},
|
||||
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true},
|
||||
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false},
|
||||
{"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false},
|
||||
{"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true},
|
||||
{"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false},
|
||||
{"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false},
|
||||
} {
|
||||
p := newPasser(kvSet{}, time.Now())
|
||||
if row.inPart {
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.lease != "" {
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != buckets.Grants {
|
||||
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if row.inController {
|
||||
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != row.bucket {
|
||||
t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket)
|
||||
}
|
||||
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
|
||||
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putModuleRaw takes a module value encoded as the catalog's modules say it into the world as branch's catalog half holds it.
|
||||
func putModuleRaw(w *world, branch, module string, raw []byte) {
|
||||
v, err := meshcatalogclient.DecodeModule(raw)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil)
|
||||
}
|
||||
|
||||
// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the
|
||||
// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main".
|
||||
func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.setTrunk("develop")
|
||||
for _, b := range []string{"beta", "develop", "alpha"} {
|
||||
w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil)
|
||||
}
|
||||
if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" {
|
||||
t.Errorf("branches = %v, want the first watched first, then by name", got)
|
||||
}
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
src, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(src), `"main"`) {
|
||||
t.Errorf("%s names the branch main in code (design 47 G §507)", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,107 +3,20 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk => ../mesh-delivery-sdk
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient => ../mesh-delivery-contracts/go/clients/meshcatalogclient
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient => ../mesh-delivery-contracts/go/clients/meshdeliveryclient
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog => ../mesh-delivery-contracts/go/seats/meshcatalog
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery => ../mesh-delivery-contracts/go/seats/meshdelivery
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks => ./asks
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents => ../mesh-delivery-teller/tellerevents
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate => ../mesh-delivery-view/viewstate
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
|
||||
|
||||
require git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient => ../mesh-delivery-contracts/go/clients/meshpipelinerunnerclient
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner => ../mesh-delivery-contracts/go/seats/meshpipelinerunner
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -21,5 +23,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
|
||||
func assignerAsk(asking int, a oc.Ask) asks.Ask {
|
||||
a.Headline, a.Explanation, a.Details, a.OnExpiry = said(a.Headline), said(a.Explanation), said(a.Details), said(a.OnExpiry)
|
||||
for i := range a.Options {
|
||||
a.Options[i].Label, a.Options[i].Does = said(a.Options[i].Label), said(a.Options[i].Does)
|
||||
}
|
||||
raw, err := json.Marshal(a)
|
||||
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
|
||||
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
|
||||
}
|
||||
return asks.Ask{Asking: asking, Ask: raw}
|
||||
}
|
||||
|
||||
// longHex is a digest or fingerprint whole: 64 hexadecimal digits.
|
||||
var longHex = regexp.MustCompile(`[0-9a-f]{64}`)
|
||||
|
||||
// said is words as an ask carries them out of the mesh: each digest or fingerprint cut to its first 12 digits, which the
|
||||
// router lets pass, where it refuses a long hexadecimal string as a secret's shape (mesh-sdk outward; novox 2026-11-16 01:24:
|
||||
// every backup ask was refused for its "(composition sha256:…)" and the fingerprints in the node's words, and the operator
|
||||
// was never asked).
|
||||
func said(words string) string {
|
||||
return longHex.ReplaceAllStringFunc(words, func(h string) string { return h[:12] })
|
||||
}
|
||||
|
||||
// AskOf is the router's ask an asks-state value carries.
|
||||
func AskOf(a asks.Ask) (oc.Ask, error) {
|
||||
var ask oc.Ask
|
||||
if err := json.Unmarshal(a.Ask, &ask); err != nil {
|
||||
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
|
||||
}
|
||||
return ask, nil
|
||||
}
|
||||
|
||||
// EncodeAsk is asks.EncodeAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
|
||||
// the ask's id is meshdeliveryclient.AskingID(key, asking), and it offers at least one option.
|
||||
func EncodeAsk(key string, a asks.Ask) ([]byte, error) {
|
||||
if _, err := checkedAsk(key, a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return asks.EncodeAsk(key, a)
|
||||
}
|
||||
|
||||
func checkedAsk(key string, a asks.Ask) (oc.Ask, error) {
|
||||
ask, err := occ.Carried(a.Ask, meshdeliveryclient.AskingID(key, a.Asking))
|
||||
if err != nil {
|
||||
return oc.Ask{}, fmt.Errorf("assign: %w", err)
|
||||
}
|
||||
return ask, nil
|
||||
}
|
||||
|
||||
// DecodeAsk reads the ask at key (asks.DecodeAsk) with the router's ask it carries.
|
||||
func DecodeAsk(key string, data []byte) (asks.Ask, oc.Ask, error) {
|
||||
a, err := asks.DecodeAsk(key, data)
|
||||
if err != nil {
|
||||
return asks.Ask{}, oc.Ask{}, err
|
||||
}
|
||||
ask, err := checkedAsk(key, a)
|
||||
if err != nil {
|
||||
return asks.Ask{}, oc.Ask{}, err
|
||||
}
|
||||
return a, ask, nil
|
||||
}
|
||||
|
||||
// ExpiresAt is when the ask ends unanswered; the zero time for an ask that names none.
|
||||
func ExpiresAt(a oc.Ask) time.Time {
|
||||
t, err := occ.Time(a.Expires)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -2,154 +2,191 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the version stays held",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
const (
|
||||
OptionRestore = "restore"
|
||||
OptionLeaveStopped = "leave-stopped"
|
||||
)
|
||||
|
||||
// The options of the outside ask (design 47 G §706).
|
||||
const (
|
||||
OptionGoOn = "go-on"
|
||||
OptionFailed = "failed"
|
||||
)
|
||||
|
||||
// OutsideAsk is the ask at a wait's bound (design 47 G §706): the move of module on node waits on cause since since, and the
|
||||
// operator says to go on (the module counts healthy there by the operator's word) or to count it as failed (held as the canary
|
||||
// rule has it). about is the ask's About, `<node>.<module>.<fingerprint hex>`.
|
||||
func OutsideAsk(key, about, node, module, fp, cause string, since time.Time, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s on %s waits on %s", module, node, cause),
|
||||
Explanation: fmt.Sprintf("The health check of %s at %s fails on %s, and the delivery has waited since %s on %s, which is outside it. "+
|
||||
"Go on counts %s healthy there by your word and the delivery goes on; count it as failed holds it as a failing check is held.",
|
||||
module, short(fp), node, since.UTC().Format(time.RFC3339), cause, module),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionGoOn, Label: "Go on", Does: module + " counts healthy on " + node + " by your word and the delivery goes on", Level: occ.Acknowledge},
|
||||
{ID: OptionFailed, Label: "Count as failed", Does: "the failing check is the delivery's and is held as the canary rule has it", Level: occ.Acknowledge},
|
||||
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
OnExpiry: "the delivery keeps waiting",
|
||||
About: about,
|
||||
})
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// WaitsOutsideFinding is the finding `delivery-waits-outside` of a wait on an outside cause (design 47 G §706): scope delivery,
|
||||
// id `<node>.<module>`, severity warning, naming the module, the machine and the cause by its condition's key, and needing
|
||||
// nothing, so it is sent to no one: the operator's one message is the ask at the bound.
|
||||
func WaitsOutsideFinding(node, module, causeKey, on string, at time.Time) Observed {
|
||||
return Observed{Scope: "delivery", ID: node + "." + module, Kind: "delivery-waits-outside", Machine: node,
|
||||
Severity: "warning",
|
||||
Summary: fmt.Sprintf("%s on %s waits on %s (the condition %s)", module, node, on, causeKey),
|
||||
Headline: fmt.Sprintf("%s on %s waits on an outside cause", module, node),
|
||||
Explanation: fmt.Sprintf("The health check of %s fails on %s while %s is open, which is outside the delivery. The delivery "+
|
||||
"waits, judges again when it clears and goes on by itself.", module, node, causeKey),
|
||||
Needs: "Nothing for you to do.", Source: "assigner.delivery-waits-outside"}
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
|
||||
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
|
||||
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
|
||||
// (raised false) once the node no longer fails alone.
|
||||
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s fails on %s alone", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
|
||||
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
|
||||
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
|
||||
node, node)}
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
|
||||
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
|
||||
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
|
||||
func NeitherLeftCondition(node, module, why string) asks.Condition {
|
||||
c := NeitherCondition(node, module, Holding{}, false)
|
||||
if why != "" {
|
||||
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
OptionNo = "no"
|
||||
OptionBackupAgain = mwc.BackupAgain
|
||||
OptionBackupSkip = mwc.BackupSkip
|
||||
OptionBackupAgain = delivery.BackupAgain
|
||||
OptionBackupSkip = delivery.BackupSkip
|
||||
)
|
||||
|
||||
// The options of the revert ask (design 47 G §710): keep the revert held, or start the reverted build on the live data.
|
||||
const (
|
||||
OptionRevertHold = "hold"
|
||||
OptionRevertLive = mwc.RevertLive
|
||||
)
|
||||
|
||||
// KindRevertHeld is the finding of a held revert (design 47 G §710), scope delivery, id `<node>.<module>`.
|
||||
const KindRevertHeld = "revert-held"
|
||||
|
||||
// RevertAsk is the ask about a revert of module on node to the build fp that the node holds with no restore point (design 47 G
|
||||
// §710, ADR 0539 §16): keep it held, or start the reverted build on the live data as it is. kept names the kept data whose form
|
||||
// the revert crosses, form its form, and words are the node's reason in its own words, from its have; asking is the number of
|
||||
// the asking, 0 the first, as the backup ask's.
|
||||
func RevertAsk(key, node, module, fp, kept, form, words string, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The revert of %s on %s has no restore point", module, node),
|
||||
Explanation: fmt.Sprintf("The revert of %s on %s to %s takes its kept data %s back across form %s, and the node holds it: %s. "+
|
||||
"%s keeps running its newer build until you answer. Started on the live data, the older build reads data written in the newer form.",
|
||||
module, node, short(fp), kept, form, words, module),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionRevertHold, Label: "Keep it held", Does: "the node keeps running the newer build, and the revert waits", Level: occ.Acknowledge},
|
||||
{ID: OptionRevertLive, Label: "Start on the live data", Does: "the node starts the older build on the live data as it is, with nothing restored", Level: occ.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
OnExpiry: "the module keeps running its newer build",
|
||||
About: key,
|
||||
})
|
||||
}
|
||||
|
||||
// RevertAnswerWords are the words a want written again on the operator's answer to the revert ask carries (design 47 G §710).
|
||||
func RevertAnswerWords(option string, asking int) string {
|
||||
return fmt.Sprintf("the operator answered %q to the revert ask (asking %d)", option, asking+1)
|
||||
}
|
||||
|
||||
// RevertHeldFinding is the finding `revert-held` of a revert the node holds with no restore point (design 47 G §710): scope
|
||||
// delivery, id `<node>.<module>`, a warning naming the reason, the kept data and its form, and needing nothing, so the
|
||||
// operator's one message is the ask.
|
||||
func RevertHeldFinding(node, module, kept, form, reason string) Observed {
|
||||
return Observed{Scope: "delivery", ID: node + "." + module, Kind: KindRevertHeld, Machine: node, Severity: "warning",
|
||||
Summary: fmt.Sprintf("The revert of %s on %s is held: %s (%s, form %s); the ask about it waits for your answer", module, node, reason, kept, form),
|
||||
Headline: fmt.Sprintf("The revert of %s on %s is held", module, node),
|
||||
Explanation: fmt.Sprintf("The node holds the revert of %s: %s (%s, form %s). %s keeps running its newer build until the ask is answered.",
|
||||
module, reason, kept, form, module),
|
||||
Needs: "Nothing for you to do.", Source: "assigner." + KindRevertHeld}
|
||||
}
|
||||
|
||||
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
||||
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
||||
// after the answer yes. The way the move was judged is in the ask. asking is the writing's number (AskingSeconds), so a key taken
|
||||
// away as stale and written again is asked again under a new id (issue 1966: nats on novox was asked once, taken back and never
|
||||
// asked again, for the rewritten key carried the id the router already kept).
|
||||
func RecordAsk(key, node, module, fp string, need RecordNeed, asking int, now time.Time) asks.Ask {
|
||||
what := map[Judgement]string{
|
||||
JudgedRecreates: "recreates a container",
|
||||
JudgedMovesData: "moves the module's data",
|
||||
JudgedCannot: "cannot be judged: " + need.Words,
|
||||
// after the answer yes. The way the move was judged is in the ask.
|
||||
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
||||
}[need.Judgement]
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
||||
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: occ.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: occ.Acknowledge},
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// AskingSeconds is the number of an asking written at now: its time in seconds, as a hold's and an assignments' ask are numbered
|
||||
// (never below 2, which AskingID needs for an id of its own). A key written again after it was taken away is another asking.
|
||||
func AskingSeconds(now time.Time) int { return max(int(now.Unix()), 2) }
|
||||
|
||||
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
||||
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
|
||||
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
|
||||
func NextBackupAsking(after int) int {
|
||||
if after < 0 {
|
||||
return 0
|
||||
@@ -157,6 +194,18 @@ func NextBackupAsking(after int) int {
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a delivery.AssignerAsk) string {
|
||||
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
@@ -164,36 +213,36 @@ func BackupAnswerWords(option string, asking int) string {
|
||||
}
|
||||
|
||||
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
||||
// without a backup, at the build fp the want names. asking is the number of the asking, 0 the first; words are the
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, words string, asking int, now time.Time) asks.Ask {
|
||||
return assignerAsk(asking, oc.Ask{
|
||||
ID: meshdeliveryclient.AskingID(key, asking),
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer.",
|
||||
module, short(fp), node, words, module),
|
||||
Options: []oc.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: occ.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: occ.Acknowledge},
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: occ.Operator,
|
||||
Expires: occ.FormatTime(now.Add(AskLasts)),
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module keeps running what it has",
|
||||
About: key,
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
||||
// naming the node, the module and the backup's words; cleared under the same key.
|
||||
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
|
||||
key := assignment.Module() + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
|
||||
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
||||
needs := "answer whether to back up again or go on without a backup"
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: occ.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
}
|
||||
|
||||
@@ -1,259 +1,148 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// 1297/B9 (the assigner's half): the backup ask is a valid ask of the router and of mesh-sdk's layout at its key, of a kind the
|
||||
// view may answer; the hold and restore asks went with the hold of a fingerprint (design 47 H §750, H-A-55).
|
||||
func TestTheAssignersAsks(t *testing.T) {
|
||||
t.Run("backup", theBackupAskIsValid)
|
||||
t.Run("ended at the channel", anAskEndedAtTheChannelIsWrittenAgain)
|
||||
t.Run("ended at the channel, read at bind", anAskEndedAtTheChannelIsReadAtBind)
|
||||
}
|
||||
|
||||
// Issue 1966, note 151: an ask the teller says ended at the operator channel without an answer (its event ask-ended) is
|
||||
// written again under a new asking, an id of its own and a new expiry, one row per kind: the rules' asks are; the
|
||||
// authorising ask of a proposal (ADR 0567 §5, §11) and a put's record ask (ADR 0546) are not, nor an asking the key no longer
|
||||
// holds, an end asking again would not change, or a body that does not read.
|
||||
func anAskEndedAtTheChannelIsWrittenAgain(t *testing.T) {
|
||||
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
|
||||
now := asked.Add(8 * 24 * time.Hour)
|
||||
digest := "sha256:" + strings.Repeat("e", 64)
|
||||
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
|
||||
revertKey, _ := asks.RevertAskKey("novox", "nats", fNew)
|
||||
recordKey, _ := asks.RecordAskKey("novox", "nats", digest)
|
||||
assignmentsKey, _ := asks.AssignmentsAskKey("p-20261028t183000-1a2b3c")
|
||||
need := RecordNeed{Node: "novox", Digest: digest, Judgement: JudgedMovesData}
|
||||
record := RecordAsk(recordKey, "novox", "nats", fNew, need, 0, asked) // written before #195: no asking
|
||||
put := puts.Put{FP: fNew, Why: "w", At: asked, Step: puts.Person, Ask: recordKey}
|
||||
byPut := RecordAsk(recordKey, "novox", "nats", fNew, need, PutAsking(put), asked)
|
||||
other := puts.Put{FP: fNew, Why: "w", At: asked.Add(time.Hour), Step: puts.Person, Ask: recordKey}
|
||||
assignments := assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(assignmentsKey, AskingSeconds(asked)), Headline: "h",
|
||||
Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(occ.ApproveLasts)), OnExpiry: "o",
|
||||
Options: []oc.Option{{ID: OptionYes, Label: "Yes", Does: "d", Level: occ.Approve}}})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
key string
|
||||
a *asks.Ask // the ask the key holds; nil for none
|
||||
ended string // the id of the asking that ended; "" for the one the key holds
|
||||
outcome string
|
||||
put *puts.Put
|
||||
why string // "" when it is written again
|
||||
}{
|
||||
{name: "backup", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeCancelled},
|
||||
{name: "revert", key: revertKey, a: ptr(assignerAsk(AskingSeconds(asked), oc.Ask{ID: meshdeliveryclient.AskingID(revertKey, AskingSeconds(asked)),
|
||||
Headline: "h", Explanation: "e", Who: occ.Operator, Expires: occ.FormatTime(asked.Add(AskLasts)), OnExpiry: "o",
|
||||
Options: []oc.Option{{ID: "hold", Label: "Hold", Does: "d", Level: occ.Acknowledge}}})), outcome: occ.OutcomeExpired},
|
||||
{name: "record", key: recordKey, a: &record, outcome: occ.OutcomeCancelled},
|
||||
{name: "record of another put", key: recordKey, a: &record, outcome: occ.OutcomeExpired, put: &other},
|
||||
{name: "record a put asked", key: recordKey, a: &byPut, outcome: occ.OutcomeExpired, put: &put, why: "put"},
|
||||
{name: "assignments", key: assignmentsKey, a: &assignments, outcome: occ.OutcomeExpired, why: "ADR 0567"},
|
||||
{name: "another asking stands", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", AskingSeconds(now), now)),
|
||||
ended: meshdeliveryclient.AskingID(backupKey, AskingSeconds(asked)), outcome: occ.OutcomeExpired, why: "holds the asking"},
|
||||
{name: "taken away", key: backupKey, ended: "backup-x", outcome: occ.OutcomeExpired, why: "taken away"},
|
||||
{name: "refused", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeRefused, why: "refused"},
|
||||
{name: "replaced", key: backupKey, a: ptr(BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked)), outcome: occ.OutcomeReplaced, why: "replaced"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var current []byte
|
||||
ended := c.ended
|
||||
if c.a != nil {
|
||||
var err error
|
||||
if current, err = EncodeAsk(c.key, *c.a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ended == "" {
|
||||
ended = mustAsk(t, *c.a).ID
|
||||
}
|
||||
}
|
||||
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: c.key, Ask: ended, Outcome: c.outcome})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, err := tellerevents.DecodeEnded(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, why := AskedAgain(e, current, c.put, now)
|
||||
if c.why != "" {
|
||||
if value != nil || !strings.Contains(why, c.why) {
|
||||
t.Fatalf("AskedAgain = %s, %q; want nothing written, %q", value, why, c.why)
|
||||
}
|
||||
return
|
||||
}
|
||||
if why != "" {
|
||||
t.Fatalf("AskedAgain says %q, want the ask written again", why)
|
||||
}
|
||||
a, ask, err := DecodeAsk(c.key, value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Asking <= c.a.Asking || ask.ID != meshdeliveryclient.AskingID(c.key, a.Asking) || ask.ID == ended {
|
||||
t.Errorf("asking %d under %s, want a new asking above %d under an id of its own, not %s", a.Asking, ask.ID, c.a.Asking, ended)
|
||||
}
|
||||
if err := occ.Check(ask, now); err != nil || !ExpiresAt(ask).Equal(now.Add(AskLasts).Truncate(time.Second)) {
|
||||
t.Errorf("the router refuses the ask written again (%v), or it lapses at %s, want %s", err, ExpiresAt(ask), now.Add(AskLasts))
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, raw := range []string{`{"v":1,"key":"k","ask":"a"}`, `{"v":1,"key":"k","ask":"a","outcome":"expired","more":1}`} {
|
||||
if _, err := tellerevents.DecodeEnded([]byte(raw)); err == nil {
|
||||
t.Errorf("DecodeEnded(%s) takes a body that is not an ended ask", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1966 note 155, ADR 0668 §4: an ended ask the teller said before the assigner's consumer existed is not delivered to
|
||||
// it, so at bind the assigner reads the operator-channel seat's asks: the query reaches back to the earliest held asking,
|
||||
// and of the answer only the held askings that lapsed or were cancelled are taken, as the teller's event says them; an
|
||||
// answered or another asker's ask, and one the key no longer holds, are not.
|
||||
func anAskEndedAtTheChannelIsReadAtBind(t *testing.T) {
|
||||
asked := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC)
|
||||
backupKey, _ := asks.BackupAskKey("novox", "nats", fNew)
|
||||
holdKey, _ := asks.BackupAskKey("g14", "nats", fNew)
|
||||
restoreKey, _ := asks.BackupAskKey("ace", "nats", fNew)
|
||||
hold := BackupAsk(holdKey, "g14", "nats", fNew, "w", AskingSeconds(asked), asked)
|
||||
restore := BackupAsk(restoreKey, "ace", "nats", fNew, "w", AskingSeconds(asked.Add(time.Hour)), asked.Add(time.Hour))
|
||||
backup := BackupAsk(backupKey, "novox", "nats", fNew, "w", 0, asked.Add(-time.Hour)) // no asking
|
||||
held := map[string][]byte{}
|
||||
for key, a := range map[string]asks.Ask{holdKey: hold, restoreKey: restore, backupKey: backup} {
|
||||
v, err := EncodeAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held[key] = v
|
||||
}
|
||||
if _, ok := ChannelAsksQuery(nil); ok {
|
||||
t.Error("with no ask held the channel is asked")
|
||||
}
|
||||
q, ok := ChannelAsksQuery(held)
|
||||
if !ok || q.State != "ended" || q.Bound != ChannelAsksBound || q.Since != asked.Add(-time.Hour-time.Minute).Format(time.RFC3339) {
|
||||
t.Errorf("the query is %v, want the ended asks since %s", q, asked.Add(-time.Hour-time.Minute).Format(time.RFC3339))
|
||||
}
|
||||
at := func(key string, a asks.Ask, state string) string {
|
||||
return fmt.Sprintf(`{"key":"t.%s","state":%q}`, mustAsk(t, a).ID, state)
|
||||
}
|
||||
answer := `{"asks":[` + strings.Join([]string{at(holdKey, hold, occ.OutcomeExpired), at(restoreKey, restore, occ.OutcomeChosen),
|
||||
at(backupKey, backup, occ.OutcomeCancelled), `{"key":"t.hold-gone","state":"expired"}`, `{"key":"open","state":"open"}`}, ",") + `],"cut":"c"}`
|
||||
ended, cut, err := EndedAtChannel(held, []byte(answer))
|
||||
want := []tellerevents.Ended{{Key: backupKey, Ask: mustAsk(t, backup).ID, Outcome: occ.OutcomeCancelled},
|
||||
{Key: holdKey, Ask: mustAsk(t, hold).ID, Outcome: occ.OutcomeExpired}}
|
||||
slices.SortFunc(want, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
|
||||
if err != nil || cut != "c" || !slices.Equal(ended, want) {
|
||||
t.Errorf("EndedAtChannel = %+v, %q, %v; want %+v and the answer's cut", ended, cut, err, want)
|
||||
}
|
||||
if _, _, err := EndedAtChannel(held, []byte(`[1]`)); err == nil {
|
||||
t.Error("an answer that does not read is taken")
|
||||
}
|
||||
}
|
||||
|
||||
func ptr(a asks.Ask) *asks.Ask { return &a }
|
||||
|
||||
func theBackupAskIsValid(t *testing.T) {
|
||||
key, err := asks.BackupAskKey("ace", "mail", fNew)
|
||||
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its key, of a kind the view may answer.
|
||||
func TestTheHoldAskIsValid(t *testing.T) {
|
||||
key, err := delivery.HoldAskKey("mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := BackupAsk(key, "ace", "mail", fNew, "health failed, phase running: connection refused", 0, now)
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// The ask names the machine whose backup failed and the backup's words.
|
||||
if !strings.Contains(mustAsk(t, a).Explanation, "on ace") || !strings.Contains(mustAsk(t, a).Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", mustAsk(t, a).Explanation)
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
// An ask written again is another asking, under an id of its own.
|
||||
again := BackupAsk(key, "ace", "mail", fNew, "x", 7, now)
|
||||
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
|
||||
t.Errorf("the asking again = %d %s", again.Asking, mustAsk(t, again).ID)
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||
}
|
||||
if _, err := EncodeAsk(key, again); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
}
|
||||
if _, err := viewstate.ViewAnswerKey(key); err != nil {
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
|
||||
t.Run("EncodeAsk refuses what mesh-sdk no longer does", func(t *testing.T) {
|
||||
good := BackupAsk(key, "ace", "mail", fNew, "x", 0, time.Unix(100, 0))
|
||||
other := good
|
||||
other.Asking = 3 // its id is the asking 0's
|
||||
empty := good
|
||||
ask := mustAsk(t, good)
|
||||
ask.Options = nil
|
||||
empty.Ask = assignerAsk(0, ask).Ask
|
||||
for name, a := range map[string]asks.Ask{"another id": other, "no option": empty, "no ask": {}} {
|
||||
if _, err := EncodeAsk(key, a); err == nil {
|
||||
t.Errorf("%s: written", name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 1952/WM6: the assigner's conditions are findings.
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
|
||||
func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range a.Ask.Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
// findings bucket cannot be written to at all.
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear; it is said straight to the router while the take-over
|
||||
// key follows the controller, and only as a finding from the take-over.
|
||||
func saidConditionsAreFindings(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
raised, cleared oc.Condition
|
||||
raised, cleared asks.Condition
|
||||
key string
|
||||
}{
|
||||
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
puts, deletes, said := map[string]mcseat.Finding{}, []string{}, 0
|
||||
f := Findings{Put: func(f mcseat.Finding) error {
|
||||
puts[views.FindingKey(f.Scope, f.ID, f.Kind)] = f
|
||||
return nil
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(oc.Condition) error { said++; return nil })
|
||||
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
|
||||
keeper := false
|
||||
f := Findings{Put: func(k string, v []byte) error {
|
||||
got, err := health.DecodeFinding(v)
|
||||
puts[k] = got
|
||||
return err
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(asks.Condition) error { said++; return nil })
|
||||
if err := say(c.raised); err != nil {
|
||||
t.Fatalf("%s: %v", c.key, err)
|
||||
}
|
||||
got, ok := puts[c.key]
|
||||
if !ok || got.Severity != c.raised.Severity || got.Headline != c.raised.Headline || got.Observer != assignment.Module() ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
|
||||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 1 {
|
||||
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
|
||||
}
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
|
||||
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
if err := (Findings{}).Say(func(oc.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
|
||||
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
|
||||
keeper = true
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 {
|
||||
t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
}
|
||||
if _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
|
||||
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
|
||||
t.Error("a condition not the assigner's is a finding")
|
||||
}
|
||||
}
|
||||
@@ -262,132 +151,103 @@ func saidConditionsAreFindings(t *testing.T) {
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
now := time.Now()
|
||||
rkey, err := asks.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
|
||||
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, 2, now)
|
||||
bkey, err := asks.BackupAskKey("ace", "mail", fNew)
|
||||
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
|
||||
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, "no space left", 0, now)
|
||||
vkey, err := asks.RevertAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := RevertAsk(vkey, "ace", "mail", fNew, "records", "3", "restore point missing: none of form 3", 0, now)
|
||||
for key, a := range map[string]asks.Ask{rkey: r, bkey: b, vkey: v} {
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("%s: the router refuses the ask: %v", key, err)
|
||||
}
|
||||
if _, err := EncodeAsk(key, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("%s: cannot be written: %v", key, err)
|
||||
}
|
||||
if _, err := viewstate.ViewAnswerKey(key); err != nil {
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("%s: the view may not answer it: %v", key, err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", mustAsk(t, r))
|
||||
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", r.Ask)
|
||||
}
|
||||
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", mustAsk(t, b))
|
||||
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", b.Ask)
|
||||
}
|
||||
// the id binds the ask to its number (review round 1, item 4); its key binds it to the build (design 47 H §750: a want names no composition)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, "x", NextBackupAsking(0), now); mustAsk(t, again).ID == mustAsk(t, b).ID {
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
// novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole.
|
||||
held := BackupAsk(bkey, "novox", "mail", fNew, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
|
||||
if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) {
|
||||
t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation)
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
}
|
||||
// design 47 G §710: the revert ask names the build, the kept data, its form and the node's words, and offers hold and live;
|
||||
// its finding is delivery.<node>.<module>.revert-held, a warning that needs nothing, the ask being the one message.
|
||||
if va := mustAsk(t, v); va.Headline != "The revert of mail on ace has no restore point" || va.Options[0].ID != OptionRevertHold ||
|
||||
va.Options[1].ID != mwc.RevertLive || len(va.Options) != 2 || va.OnExpiry != "the module keeps running its newer build" ||
|
||||
!strings.Contains(va.Explanation, "records") || !strings.Contains(va.Explanation, "form 3") || !strings.Contains(va.Explanation, short(fNew)) ||
|
||||
!strings.Contains(va.Explanation, "Started on the live data, the older build reads data written in the newer form.") {
|
||||
t.Errorf("the revert ask = %+v", va)
|
||||
}
|
||||
if f := RevertHeldFinding("ace", "mail", "records", "3", "restore point missing").Finding(now); views.FindingKey(f.Scope, f.ID, f.Kind) != "delivery.ace.mail.revert-held" ||
|
||||
f.Severity != "warning" || f.Needs != "Nothing for you to do." ||
|
||||
f.Summary != "The revert of mail on ace is held: restore point missing (records, form 3); the ask about it waits for your answer" {
|
||||
t.Errorf("the revert finding = %+v", f)
|
||||
}
|
||||
if RevertAnswerWords(OptionRevertLive, 1) != `the operator answered "live" to the revert ask (asking 2)` {
|
||||
t.Errorf("the words of a live answer: %q", RevertAnswerWords(OptionRevertLive, 1))
|
||||
}
|
||||
c := BackupCondition("ace", "mail", fNew, "no space left", true)
|
||||
if _, err := occ.EncodeCondition(c); err != nil {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
|
||||
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := occ.Check(mustAsk(t, a), now); err != nil {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := EncodeAsk(rkey, a); err != nil {
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := viewstate.ViewAnswerFits(viewstate.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
|
||||
yes := mustAsk(t, a).Options[0]
|
||||
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
|
||||
// acknowledge), as it did through the sdk before.
|
||||
if viewed == nil {
|
||||
if o, ok := occ.OptionOf(mustAsk(t, a), OptionYes); ok && o.Level != "" && o.Level != occ.Acknowledge {
|
||||
viewed = fmt.Errorf("only a channel that proves who answered takes the option %s", o.ID)
|
||||
}
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(mustAsk(t, a).Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, mustAsk(t, a).Explanation)
|
||||
case bus && (yes.Level != occ.Approve || yes.Binds == "" || viewed == nil || expiresAt(t, a).After(now.Add(occ.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, expiresAt(t, a))
|
||||
case !bus && (yes.Level != occ.Acknowledge || viewed != nil):
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if mustAsk(t, b).ID == mustAsk(t, a).ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, mustAsk(t, b).ID, mustAsk(t, a).ID)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := occ.Performs(yes, PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mustAsk is the router's ask an asks-state value carries.
|
||||
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
|
||||
t.Helper()
|
||||
ask, err := AskOf(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
return ask
|
||||
}
|
||||
|
||||
func expiresAt(t *testing.T, a asks.Ask) time.Time {
|
||||
t.Helper()
|
||||
return ExpiresAt(mustAsk(t, a))
|
||||
}
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"strings"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
)
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-module-assigner while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) oc.Condition {
|
||||
key := assignment.Module() + ".seat-states"
|
||||
subject := oc.ConditionSubject{Scope: "module", ID: assignment.Module()}
|
||||
if len(unbound) == 0 {
|
||||
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared, Resolved: "the seat mesh-module-assigner holds all its states."}
|
||||
}
|
||||
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
|
||||
return oc.Condition{Key: key, Severity: occ.SeverityWarning, Subject: subject, State: occ.StateRaised,
|
||||
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
|
||||
Explanation: occ.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
|
||||
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
|
||||
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
|
||||
// and writes nothing itself: the program hands it what it watched and writes what it answers.
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
|
||||
const SDKRepository = "novox/mesh-sdk"
|
||||
|
||||
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
|
||||
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
|
||||
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
|
||||
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
|
||||
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
|
||||
"node-engine's validator would refuse"
|
||||
|
||||
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
|
||||
// why, and each node's open assignments.
|
||||
type Mesh struct {
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Assigned map[string][]string
|
||||
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
|
||||
// composer at another mesh-sdk reads (issue 1622).
|
||||
Digests map[string]string
|
||||
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
|
||||
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
|
||||
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
|
||||
Input *compose.Input
|
||||
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
|
||||
// nodes and their assignments alone, and says so in its warnings.
|
||||
InputUnread string
|
||||
}
|
||||
|
||||
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
|
||||
// member (empty when none), and how many modules it rebuilds.
|
||||
type Change struct {
|
||||
Group, Heads string
|
||||
Manifests map[string]catalogue.Manifest
|
||||
Unreadable map[string]string
|
||||
Removed []string
|
||||
Twice map[string][]string
|
||||
SDKHead string
|
||||
Rebuilt int
|
||||
// Digests is the manifest digest of every module the change moves or adds, readable by the assigner or not, and
|
||||
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
|
||||
Digests map[string]string
|
||||
Members map[string]string
|
||||
// Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR
|
||||
// 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named.
|
||||
Held []delivery.KeptValue
|
||||
}
|
||||
|
||||
// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another
|
||||
// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key.
|
||||
func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue {
|
||||
var out []delivery.KeptValue
|
||||
for _, raw := range kept {
|
||||
var k delivery.KeptValue
|
||||
if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" {
|
||||
continue
|
||||
}
|
||||
if head, ok := heads[k.Pull]; ok && head == k.Head {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers
|
||||
// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the
|
||||
// declaration refuses, or a key it does not declare, is a failure.
|
||||
func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) {
|
||||
for _, k := range held {
|
||||
var v any
|
||||
if err := json.Unmarshal(k.Value, &v); err != nil {
|
||||
said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err))
|
||||
continue
|
||||
}
|
||||
d, declared := in.Catalogue[k.Module].Settings[k.Key]
|
||||
if !declared {
|
||||
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull))
|
||||
continue
|
||||
}
|
||||
if problems := catalogue.ValueProblems(d, v); len(problems) > 0 {
|
||||
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module,
|
||||
k.Node, k.Pull, strings.Join(problems, "; ")))
|
||||
continue
|
||||
}
|
||||
at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node })
|
||||
if at < 0 {
|
||||
said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node))
|
||||
continue
|
||||
}
|
||||
if in.Nodes[at].Settings == nil {
|
||||
in.Nodes[at].Settings = map[string][]catalogue.Layer{}
|
||||
}
|
||||
in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}})
|
||||
said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved))
|
||||
}
|
||||
return said, fails
|
||||
}
|
||||
|
||||
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
|
||||
// mesh-placement judges it.
|
||||
func Touched(m Mesh, c Change) []string {
|
||||
moved := map[string]bool{}
|
||||
for name := range c.Manifests {
|
||||
moved[name] = true
|
||||
}
|
||||
for name := range c.Unreadable {
|
||||
moved[name] = true
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
moved[name] = true
|
||||
}
|
||||
provided, required := map[string]bool{}, map[string]bool{}
|
||||
for name := range moved {
|
||||
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
|
||||
for _, o := range man.Provides {
|
||||
provided[o.Name] = true
|
||||
}
|
||||
}
|
||||
for _, r := range c.Manifests[name].Requires {
|
||||
required[r] = true
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
|
||||
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
|
||||
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
|
||||
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
|
||||
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
|
||||
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
|
||||
in := compose.Input{Catalogue: catalog}
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
for _, mod := range assigned[n] {
|
||||
for _, cl := range catalog[mod].Claims {
|
||||
scope := cl.Scope
|
||||
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||
scope = seat.Scope
|
||||
}
|
||||
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
|
||||
}
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
|
||||
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
|
||||
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
|
||||
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
|
||||
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
|
||||
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
|
||||
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
|
||||
var fails []string
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Unreadable) {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
trunk := input(m.Manifests, m.Assigned)
|
||||
changed := map[string]catalogue.Manifest{}
|
||||
for name, man := range m.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for name, man := range c.Manifests {
|
||||
changed[name] = man
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(changed, name)
|
||||
}
|
||||
after := input(changed, m.Assigned)
|
||||
laid, heldFails := layHeld(&after, c.Held)
|
||||
v.Warnings = append(v.Warnings, laid...)
|
||||
for _, f := range heldFails {
|
||||
failAt("", f)
|
||||
}
|
||||
for _, node := range machines {
|
||||
_, before := compose.Resolve(trunk, node)
|
||||
_, now := compose.Resolve(after, node)
|
||||
switch {
|
||||
case now == nil:
|
||||
case before != nil:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
|
||||
}
|
||||
}
|
||||
if c.SDKHead != "" {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
|
||||
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// finish adds what every composition says beside its own judgement (more modules rebuilt than CompositionRebuildWarning,
|
||||
// what is not judged, the machines not composed, a change that touches none) and its state from fails.
|
||||
func finish(v delivery.CompositionVerdict, m Mesh, c Change, machines []string, fails []string) delivery.CompositionVerdict {
|
||||
if c.Rebuilt > delivery.CompositionRebuildWarning {
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
|
||||
}
|
||||
v.Warnings = append(v.Warnings, NotJudged)
|
||||
if could := CouldRun(m, c, machines); len(could) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
|
||||
}
|
||||
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
|
||||
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
|
||||
}
|
||||
if len(fails) > 0 {
|
||||
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
|
||||
return v
|
||||
}
|
||||
v.State = delivery.CompositionPass
|
||||
return v
|
||||
}
|
||||
|
||||
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
|
||||
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
|
||||
// the facts snapshot), so each is named as not judged.
|
||||
func CouldRun(m Mesh, c Change, composed []string) []string {
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for name := range c.Manifests {
|
||||
if !slices.Contains(modules, name) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
|
||||
func grantsOf(man catalogue.Manifest) string {
|
||||
var offers []string
|
||||
for _, o := range man.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
|
||||
}
|
||||
|
||||
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
|
||||
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
|
||||
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
|
||||
func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||
provisions := map[string]bool{}
|
||||
altered := false
|
||||
for name, man := range c.Manifests {
|
||||
before, had := m.Manifests[name]
|
||||
if had && grantsOf(before) == grantsOf(man) {
|
||||
continue
|
||||
}
|
||||
altered = true
|
||||
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
|
||||
provisions[r] = true
|
||||
}
|
||||
}
|
||||
if !altered {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for node, modules := range m.Assigned {
|
||||
if slices.Contains(composed, node) {
|
||||
continue
|
||||
}
|
||||
for _, mod := range modules {
|
||||
man := m.Manifests[mod]
|
||||
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
|
||||
out = append(out, node)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// withOverlay is base with every manifest of over laid over it, a copy.
|
||||
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
|
||||
out := make(map[string]catalogue.Manifest, len(base)+len(over))
|
||||
for name, man := range base {
|
||||
out[name] = man
|
||||
}
|
||||
for name, man := range over {
|
||||
out[name] = man
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func meshOf() Mesh {
|
||||
return Mesh{
|
||||
Manifests: map[string]catalogue.Manifest{
|
||||
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
|
||||
"web": {Module: "web", Version: "1"},
|
||||
},
|
||||
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
|
||||
}
|
||||
}
|
||||
|
||||
func hasWarning(v delivery.CompositionVerdict, part string) bool {
|
||||
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
|
||||
}
|
||||
|
||||
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
|
||||
// the machine and the mesh-sdk it composed with, and says what it did not judge.
|
||||
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "v0.1.14-x", 0)
|
||||
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
|
||||
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
|
||||
}
|
||||
if !hasWarning(v, "issue 1461") {
|
||||
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
// 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged
|
||||
// against the head's declaration; one of another pull request or another head is not laid.
|
||||
web, err := catalogue.ParseManifest([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := map[string][]byte{}
|
||||
for i, k := range []delivery.KeptValue{
|
||||
{Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"},
|
||||
{Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)},
|
||||
{Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)},
|
||||
} {
|
||||
kept[strconv.Itoa(i)], _ = json.Marshal(k)
|
||||
}
|
||||
held := HeldOf(kept, map[string]string{"novox/x#1": "h1"})
|
||||
if len(held) != 1 || string(held[0].Value) != "81" {
|
||||
t.Fatalf("held = %+v, want #1's at its head alone", held)
|
||||
}
|
||||
c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held}
|
||||
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") {
|
||||
t.Errorf("composition = %+v", v)
|
||||
}
|
||||
c.Held[0].Value = json.RawMessage(`"eighty"`)
|
||||
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") {
|
||||
t.Errorf("a held value the declaration refuses: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||
// consumer's machine is touched when its provider changes.
|
||||
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
|
||||
func TestUnreadableTwiceAndWide(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
|
||||
v = Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
|
||||
t.Fatalf("composition = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
|
||||
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
|
||||
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
|
||||
v := Compose(meshOf(), c, "v0.1.14-own", 0)
|
||||
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
|
||||
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
|
||||
}
|
||||
c.SDKHead = ""
|
||||
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
|
||||
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
|
||||
// assigned nowhere touches none, and the verdict says so.
|
||||
func TestTouched(t *testing.T) {
|
||||
m := meshOf()
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
|
||||
t.Errorf("touched by store = %v", got)
|
||||
}
|
||||
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
|
||||
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||
}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
|
||||
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
|
||||
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
|
||||
// names none of the second.
|
||||
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
|
||||
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
|
||||
m.Assigned["novox"] = []string{"bus"}
|
||||
m.Assigned["shanks"] = []string{"db"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
|
||||
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
|
||||
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
|
||||
}
|
||||
if !hasWarning(v, "whose grants alone the change alters: novox") {
|
||||
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
|
||||
}
|
||||
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
|
||||
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
|
||||
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package assign
|
||||
|
||||
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
|
||||
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
|
||||
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
|
||||
// judgement); neither and unchanged are the assigner's own.
|
||||
type Judgement = string
|
||||
|
||||
// The judgements of a move.
|
||||
const (
|
||||
// JudgedRecreates is a move that recreates a container.
|
||||
JudgedRecreates = nec.JudgedRecreates
|
||||
// JudgedMovesData is a move that gives the module's data another place.
|
||||
JudgedMovesData = nec.JudgedMovesData
|
||||
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
|
||||
JudgedNeither = "neither"
|
||||
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
|
||||
JudgedCannot = nec.JudgedCannot
|
||||
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
|
||||
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
|
||||
// changes nothing on the node, so it needs no person's yes.
|
||||
JudgedUnchanged = "unchanged"
|
||||
)
|
||||
|
||||
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
|
||||
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
|
||||
// the module). Words say why, and are required for a move that cannot be judged.
|
||||
//
|
||||
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
|
||||
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
|
||||
type ComposedDigest struct {
|
||||
V int `json:"v"`
|
||||
Digest string `json:"digest"`
|
||||
Judgement Judgement `json:"judgement"`
|
||||
Against string `json:"against,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
|
||||
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
|
||||
// record hold (ADR 0539 clause 7).
|
||||
UserListOnly bool `json:"user-list-only,omitempty"`
|
||||
Policy string `json:"policy,omitempty"`
|
||||
PolicyFrom string `json:"policy-from,omitempty"`
|
||||
PolicyWhy string `json:"policy-why,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The cases of a group's composition at its mesh-sdk head: the trunk as it is, and the trunk with the change.
|
||||
const (
|
||||
CaseTrunk = "trunk"
|
||||
CaseChange = "change"
|
||||
)
|
||||
|
||||
// SDKDir is the directory of mesh-sdk's Go module in its repository, where the composer is built at a head; a reader's
|
||||
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
|
||||
const SDKDir = "go"
|
||||
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
|
||||
// them, the node-engine applies them. Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
|
||||
|
||||
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
|
||||
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
|
||||
// them, every manifest named by its digest in the indexer's bucket, the seats held from the claims (the assigner's input).
|
||||
func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
|
||||
trunk, change := map[string]composer.Manifest{}, map[string]composer.Manifest{}
|
||||
for name, digest := range m.Digests {
|
||||
trunk[name] = composer.Manifest{Digest: digest}
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for name, digest := range c.Digests {
|
||||
change[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
for _, name := range c.Removed {
|
||||
delete(change, name)
|
||||
}
|
||||
in, fromClaims := plainInput(m.Assigned), true
|
||||
if m.Input != nil {
|
||||
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
|
||||
in = *m.Input
|
||||
in.Catalogue = nil
|
||||
fromClaims = len(in.Holdings) == 0
|
||||
}
|
||||
changeCase := composer.Case{Name: CaseChange, Manifests: change}
|
||||
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
|
||||
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
|
||||
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
|
||||
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
|
||||
}
|
||||
|
||||
// ReaderJob is the compose job that reads the manifests a group changes with the mesh-sdk a reader module pins at
|
||||
// commit (ADR 0482 clause 5, issue 1623): no machine is composed, every changed manifest is only read.
|
||||
func ReaderJob(c Change, module, repository, commit string) (delivery.ComposeJob, error) {
|
||||
changed := map[string]composer.Manifest{}
|
||||
for name, digest := range c.Digests {
|
||||
changed[name] = composer.Manifest{Digest: digest}
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: CaseChange, Manifests: changed}}}
|
||||
return job(repository, commit, delivery.RootPath, req, module+" at "+commit)
|
||||
}
|
||||
|
||||
func job(repository, commit, dir string, req composer.Request, reader string) (delivery.ComposeJob, error) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return delivery.ComposeJob{}, err
|
||||
}
|
||||
return delivery.ComposeJob{Fetch: map[string]any{"repository": repository, "commit": commit}, Dir: dir,
|
||||
Package: composer.Package, Request: raw, Reader: reader}, nil
|
||||
}
|
||||
|
||||
// plainInput is the composition's input without its catalogue: the nodes by name with what is assigned to them (the
|
||||
// composer derives the seats held from the claims, as input does).
|
||||
func plainInput(assigned map[string][]string) compose.Input {
|
||||
var in compose.Input
|
||||
nodes := make([]string, 0, len(assigned))
|
||||
for n := range assigned {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, n := range nodes {
|
||||
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// Ran is one compose ask's end: its step's state and reason, and the composer's verdict when the step passed.
|
||||
type Ran struct {
|
||||
State string
|
||||
Reason string
|
||||
Verdict *composer.Verdict
|
||||
}
|
||||
|
||||
// RanOf reads a compose ask's end from its step value; a step that passed with an answer that cannot be read is a
|
||||
// failure, named.
|
||||
func RanOf(step delivery.Step) Ran {
|
||||
r := Ran{State: step.State, Reason: step.Reason}
|
||||
if step.State != delivery.StepPassed {
|
||||
return r
|
||||
}
|
||||
var a delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &a); err != nil {
|
||||
return Ran{State: delivery.StepFailed, Reason: "the compose step's answer cannot be read: " + err.Error()}
|
||||
}
|
||||
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
|
||||
return Ran{State: delivery.StepFailed, Reason: fmt.Sprintf("the composer's verdict cannot be read (version %d): %v", v.V, err)}
|
||||
}
|
||||
r.Verdict = &v
|
||||
return r
|
||||
}
|
||||
|
||||
func caseOf(v *composer.Verdict, name string) (composer.CaseVerdict, bool) {
|
||||
if v == nil {
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name == name {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
return composer.CaseVerdict{}, false
|
||||
}
|
||||
|
||||
// Reader is one module whose mesh-sdk reads the group's manifests, at the commit judged: the group's member head when
|
||||
// the group holds one in its repository, else a build its node runs.
|
||||
type Reader struct {
|
||||
Module, Commit string
|
||||
Member bool
|
||||
Ran Ran
|
||||
}
|
||||
|
||||
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
|
||||
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
|
||||
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
|
||||
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
|
||||
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
|
||||
// said and judged nothing.
|
||||
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
|
||||
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
|
||||
machines := Touched(m, c)
|
||||
v.Machines = machines
|
||||
var fails []string
|
||||
if head.State != delivery.StepPassed {
|
||||
v.Words = fmt.Sprintf("the composer at mesh-sdk %s did not run: %s %s", c.SDKHead, head.State, head.Reason)
|
||||
v.State = delivery.CompositionFail
|
||||
return v
|
||||
}
|
||||
if head.Verdict.SDK != "" {
|
||||
v.SDK = head.Verdict.SDK
|
||||
}
|
||||
trunk, okT := caseOf(head.Verdict, CaseTrunk)
|
||||
change, okC := caseOf(head.Verdict, CaseChange)
|
||||
if !okT || !okC {
|
||||
v.State, v.Words = delivery.CompositionFail, "the composer's verdict holds no trunk or no change case"
|
||||
return v
|
||||
}
|
||||
failAt := func(machine, words string) {
|
||||
if v.Machine == "" && machine != "" {
|
||||
v.Machine = machine
|
||||
}
|
||||
fails = append(fails, words)
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if r := change.Manifests[name]; !r.Accepted {
|
||||
failAt("", fmt.Sprintf("the manifest of %s cannot be read by mesh-sdk %s: %s", name, c.SDKHead, r.Refused))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(trunk.Manifests) {
|
||||
if _, changed := c.Digests[name]; changed || slices.Contains(c.Removed, name) || trunk.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
if _, before := m.Unreadable[name]; before {
|
||||
continue // already unreadable on the trunk: not the group's doing
|
||||
}
|
||||
failAt("", fmt.Sprintf("mesh-sdk %s refuses the trunk's manifest of %s, which the group does not change: %s", c.SDKHead, name, trunk.Manifests[name].Refused))
|
||||
}
|
||||
for _, name := range sortedKeys(c.Twice) {
|
||||
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, name := range c.Removed {
|
||||
if slices.Contains(m.Assigned[node], name) {
|
||||
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||
}
|
||||
}
|
||||
before, now := trunk.Machines[node], change.Machines[node]
|
||||
switch {
|
||||
case now.Composes:
|
||||
case !before.Composes:
|
||||
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %s", node, before.Reason))
|
||||
default:
|
||||
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %s", node, now.Reason))
|
||||
}
|
||||
}
|
||||
readerFails, readerWarnings := judgeReaders(c, change, readers)
|
||||
fails = append(fails, readerFails...)
|
||||
v.Warnings = append(v.Warnings, readerWarnings...)
|
||||
if m.InputUnread != "" {
|
||||
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
|
||||
"cannot be read from the bus ("+m.InputUnread+")")
|
||||
}
|
||||
return finish(v, m, c, machines, fails)
|
||||
}
|
||||
|
||||
// NoComposer says whether a compose step failed because the mesh-sdk it was built at has no composer command (a mesh-sdk
|
||||
// from before issue 1621): Go's words for a package its module does not hold.
|
||||
func NoComposer(reason string) bool {
|
||||
for _, words := range []string{"cannot find package", "no required module provides package", "is not in std", "cannot find module providing package", "does not contain package"} {
|
||||
if strings.Contains(reason, words) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails, warnings []string) {
|
||||
for _, r := range readers {
|
||||
at := r.Module + " at " + r.Commit
|
||||
if r.Member {
|
||||
at = r.Module + "'s member head " + r.Commit
|
||||
}
|
||||
if r.Ran.State != delivery.StepPassed {
|
||||
if NoComposer(r.Ran.Reason) {
|
||||
warnings = append(warnings, fmt.Sprintf("not judged whether %s reads the group's manifests: the mesh-sdk it pins has no "+
|
||||
"composer command (before issue 1621)", at))
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins did not run: %s %s", at, r.Ran.State, r.Ran.Reason))
|
||||
continue
|
||||
}
|
||||
read, ok := caseOf(r.Ran.Verdict, CaseChange)
|
||||
if !ok {
|
||||
fails = append(fails, fmt.Sprintf("the composer of the mesh-sdk %s pins answered no change case", at))
|
||||
continue
|
||||
}
|
||||
for _, name := range sortedKeys(c.Digests) {
|
||||
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
|
||||
continue
|
||||
}
|
||||
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
|
||||
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
|
||||
}
|
||||
}
|
||||
return fails, warnings
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
var testTime = time.Date(2026, 10, 24, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
const sdkHead = "1111111111111111111111111111111111111111"
|
||||
|
||||
// raws are the manifests of meshOf as their repositories hold them, by digest, as the indexer keeps them; `web2` is web
|
||||
// with a field `shares` that the assigner's own mesh-sdk does not read (the shape of issue 1593).
|
||||
var raws = map[string]string{
|
||||
"d-mail": `{"module": "mail", "version": "1", "requires": ["postgres-database"]}`,
|
||||
"d-store": `{"module": "store", "version": "1", "provides": [{"name": "postgres-database"}]}`,
|
||||
"d-web": `{"module": "web", "version": "1"}`,
|
||||
"d-web2": `{"module": "web", "version": "2", "shares": ["photos"]}`,
|
||||
}
|
||||
|
||||
// asTheHeadReads is how mesh-sdk at the group's head reads each manifest: as the assigner's own reads it, but for the
|
||||
// field it adds, which it accepts. The head's composer is built and run by the Go build module (ADR 0482 clause 4); here
|
||||
// its verdict is made by mesh-sdk's composer.Run over the same request, with `web2` read as the head reads it.
|
||||
var asTheHeadReads = map[string]string{"d-web2": `{"module": "web", "version": "2"}`}
|
||||
|
||||
func digestedMesh() Mesh {
|
||||
m := meshOf()
|
||||
m.Digests = map[string]string{"mail": "d-mail", "store": "d-store", "web": "d-web"}
|
||||
return m
|
||||
}
|
||||
|
||||
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
|
||||
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
|
||||
func webShares(members map[string]string) Change {
|
||||
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
|
||||
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
|
||||
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
|
||||
}
|
||||
|
||||
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
|
||||
// (with read standing in for how that mesh-sdk reads a digest), run, and its verdict put in a passed step.
|
||||
func run(t *testing.T, job delivery.ComposeJob, read map[string]string) Ran {
|
||||
t.Helper()
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := composer.Fill(&req, func(d string) ([]byte, error) {
|
||||
if r, ok := read[d]; ok {
|
||||
return []byte(r), nil
|
||||
}
|
||||
return []byte(raws[d]), nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := composer.Run(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.SDK = "(devel) " + job.Fetch["commit"].(string)
|
||||
vr, _ := json.Marshal(v)
|
||||
ans, _ := json.Marshal(delivery.ComposeAnswer{Built: true, Key: "composer-0", Verdict: vr})
|
||||
return RanOf(delivery.Step{State: delivery.StepPassed, Result: ans})
|
||||
}
|
||||
|
||||
// 1622/AC1: a group holding a mesh-sdk pull request is composed by the composer built at the member's head: the job
|
||||
// fetches mesh-sdk at that head, builds mesh-sdk's composer command in its Go module, and hands it the trunk and the
|
||||
// change by digest, never the manifests' bytes, composing the machines the change touches.
|
||||
func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, err := HeadJob(digestedMesh(), c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Fetch["repository"] != SDKRepository || job.Fetch["commit"] != sdkHead || job.Dir != "go" || job.Package != composer.Package {
|
||||
t.Fatalf("job = %+v", job)
|
||||
}
|
||||
ask := delivery.StepAsk{ID: "mh3k2x9qa1b2c5", Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: testTime, Compose: &job}
|
||||
if _, err := delivery.EncodeStepAsk(ask); err != nil {
|
||||
t.Fatalf("the ask is refused by its layout: %v", err)
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(req.Cases) != 2 || req.Cases[0].Name != CaseTrunk || req.Cases[1].Name != CaseChange {
|
||||
t.Fatalf("cases = %+v", req.Cases)
|
||||
}
|
||||
if req.Cases[0].Manifests["web"].Digest != "d-web" || req.Cases[1].Manifests["web"].Digest != "d-web2" || len(req.Cases[1].Manifests["web"].Raw) != 0 {
|
||||
t.Errorf("the cases do not name web by its trunk and its change digest: %+v", req.Cases)
|
||||
}
|
||||
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
|
||||
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
|
||||
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
|
||||
m := digestedMesh()
|
||||
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
|
||||
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
|
||||
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
|
||||
placedJob, err := HeadJob(m, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := composer.Decode(placedJob.Request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
|
||||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
|
||||
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
|
||||
// verdict names the mesh-sdk the composer was built with; with the assigner's own mesh-sdk it fails.
|
||||
func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
|
||||
c := webShares(map[string]string{"novox/mesh-controller": "2222222222222222222222222222222222222222"})
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
v := FromComposer(digestedMesh(), c, head, nil, 0)
|
||||
if v.State != delivery.CompositionPass || v.SDK != "(devel) "+sdkHead || !slices.Equal(v.Machines, []string{"g14"}) {
|
||||
t.Fatalf("with the head's composer: %+v", v)
|
||||
}
|
||||
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||
t.Errorf("the verdict cannot be written: %v", err)
|
||||
}
|
||||
own := Compose(digestedMesh(), c, "v0.1.14-own", 0)
|
||||
if own.State != delivery.CompositionFail || !strings.Contains(own.Words, "shares") {
|
||||
t.Fatalf("with the assigner's own mesh-sdk: %+v, want a fail naming the field", own)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
|
||||
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
|
||||
// the SDK head (its composer reads what the head reads) it passes.
|
||||
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
running := "3333333333333333333333333333333333333333"
|
||||
rjob, err := ReaderJob(c, "mesh-controller", "novox/mesh-controller", running)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
|
||||
t.Fatalf("reader job = %+v", rjob)
|
||||
}
|
||||
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
|
||||
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
|
||||
}
|
||||
member := "2222222222222222222222222222222222222222"
|
||||
pinned := run(t, rjob, asTheHeadReads)
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: member, Member: true, Ran: pinned}}, 0)
|
||||
if v.State != delivery.CompositionPass {
|
||||
t.Fatalf("with a controller member pinned to the head: %+v, want a pass", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1623: a reader whose mesh-sdk has no composer command (before issue 1621) is said, not failed; a reader's composer that
|
||||
// failed for another cause fails the group, loud.
|
||||
func TestAReaderWithNoComposerIsSaidAndAnotherFailureFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
head := run(t, job, asTheHeadReads)
|
||||
none := Ran{State: delivery.StepFailed, Reason: "go build: no required module provides package git.novox.be/novox/mesh-sdk/go/cmd/mesh-compose"}
|
||||
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: none}}, 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "mesh-host at abc") || !hasWarning(v, "before issue 1621") {
|
||||
t.Fatalf("a reader with no composer: %+v", v)
|
||||
}
|
||||
broke := Ran{State: delivery.StepFailed, Reason: "the fetch could not reach the forge"}
|
||||
v = FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-host", Commit: "abc", Ran: broke}}, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "could not reach the forge") {
|
||||
t.Fatalf("a reader whose composer failed: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the head's composer that did not run fails the composition naming why, never passes it; a trunk manifest the
|
||||
// head's mesh-sdk refuses fails it too, since the group would deliver a reader that refuses it.
|
||||
func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
|
||||
c := webShares(nil)
|
||||
v := FromComposer(digestedMesh(), c, Ran{State: delivery.StepRefused, Reason: "the guard refuses it"}, nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "the guard refuses it") || !strings.Contains(v.Words, sdkHead) {
|
||||
t.Fatalf("a head composer that did not run: %+v", v)
|
||||
}
|
||||
job, _ := HeadJob(digestedMesh(), c)
|
||||
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
|
||||
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
|
||||
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
|
||||
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
)
|
||||
|
||||
// AskLasts is how long an ask of the assigner's rules stands at the operator channel before it lapses unanswered: the hold,
|
||||
// restore, record, backup, outside and revert asks.
|
||||
const AskLasts = 7 * 24 * time.Hour
|
||||
|
||||
// writtenAgainOnEnd is, per kind of the assigner's asks, whether an asking that ended at the channel without an answer while
|
||||
// its key stands is written again under a new asking (issue 1966, note 151), or why not. A rule's ask waits for a person
|
||||
// as long as its key stands, so it is asked again; the authorising ask of a proposal ends the proposal when it lapses
|
||||
// (ADR 0567 §5) and is asked again by the seat's verb `again` (§11).
|
||||
var writtenAgainOnEnd = map[string]string{
|
||||
asks.AskKindHold: "",
|
||||
asks.AskKindRestore: "",
|
||||
asks.AskKindBackup: "",
|
||||
asks.AskKindOutside: "", // raised again at once while the wait stands (design 47 G §706); the wait's end takes the key away
|
||||
asks.AskKindRecord: "",
|
||||
asks.AskKindRevert: "", // raised again at once while the hold stands (design 47 G §710); one answered hold is not asked again
|
||||
asks.AskKindAssignments: "a proposal's authorising ask that lapsed ends the proposal (ADR 0567 §5); the verb again asks it again (§11)",
|
||||
}
|
||||
|
||||
// AskedAgain is the ask at key written again after the teller said its asking ended at the channel without an answer (the
|
||||
// teller's event ask-ended): the value to write by compare-and-set over current, or why nothing is written. Nothing is written
|
||||
// for an outcome other than a lapse or a cancellation, for a key that holds another asking now (an answer to it, or a newer
|
||||
// asking, came since), for a kind its own rule ends (writtenAgainOnEnd), and for a record ask a put asked (put, nil for none:
|
||||
// the put at the record ask's node and module), since its put is refused when it lapses (ADR 0546). The new asking is
|
||||
// AskingSeconds(now), above the old one, under an id of its own, standing AskLasts from now.
|
||||
func AskedAgain(e tellerevents.Ended, current []byte, put *puts.Put, now time.Time) ([]byte, string) {
|
||||
if e.Outcome != occ.OutcomeExpired && e.Outcome != occ.OutcomeCancelled {
|
||||
return nil, fmt.Sprintf("it ended %s, which asking again would not change", e.Outcome)
|
||||
}
|
||||
k, err := asks.ParseAnyAskKey(e.Key)
|
||||
if err != nil {
|
||||
return nil, err.Error()
|
||||
}
|
||||
why, known := writtenAgainOnEnd[k.Kind]
|
||||
if !known {
|
||||
return nil, "the assigner asks no " + k.Kind + " ask"
|
||||
}
|
||||
if why != "" {
|
||||
return nil, why
|
||||
}
|
||||
if current == nil {
|
||||
return nil, "the key is taken away: nothing is asked"
|
||||
}
|
||||
a, ask, err := DecodeAsk(e.Key, current)
|
||||
if err != nil {
|
||||
return nil, err.Error()
|
||||
}
|
||||
if ask.ID != e.Ask {
|
||||
return nil, "the key holds the asking " + ask.ID + " now, not the one that ended"
|
||||
}
|
||||
if k.Kind == asks.AskKindRecord && put != nil && put.Ask == e.Key && PutAsking(*put) == a.Asking {
|
||||
return nil, "a put asked it, and its put is refused when it lapses (ADR 0546)"
|
||||
}
|
||||
asking := max(AskingSeconds(now), a.Asking+1)
|
||||
ask.ID, ask.Expires = meshdeliveryclient.AskingID(e.Key, asking), occ.FormatTime(now.Add(AskLasts))
|
||||
value, err := EncodeAsk(e.Key, assignerAsk(asking, ask))
|
||||
if err != nil {
|
||||
return nil, err.Error()
|
||||
}
|
||||
return value, ""
|
||||
}
|
||||
|
||||
// ChannelAsksBound is the most bytes of asks the assigner asks the operator-channel seat's verb asks for at its consumer's
|
||||
// bind: every ended asking of its held asks fits.
|
||||
const ChannelAsksBound = 1 << 20
|
||||
|
||||
// ChannelAsksQuery is the arguments of the operator-channel seat's verb asks that answer, at the assigner's consumer's bind,
|
||||
// every asking of its held asks (key to value, as the asks state holds them) that ended at the channel: the ended asks
|
||||
// opened since the earliest of them could have been opened, less a minute (issue 1966 note 155, ADR 0668 §4). ok is false
|
||||
// when no held ask reads, and then nothing is asked.
|
||||
func ChannelAsksQuery(held map[string][]byte) (query oc.AsksInput, ok bool) {
|
||||
var since time.Time
|
||||
for key, value := range held {
|
||||
a, ask, err := DecodeAsk(key, value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
opened := ExpiresAt(ask).Add(-AskLasts)
|
||||
if a.Asking >= 2 {
|
||||
if at := time.Unix(int64(a.Asking), 0); at.Before(opened) {
|
||||
opened = at
|
||||
}
|
||||
}
|
||||
if !ok || opened.Before(since) {
|
||||
since, ok = opened, true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return oc.AsksInput{}, false
|
||||
}
|
||||
return oc.AsksInput{State: "ended", Since: since.Add(-time.Minute).UTC().Format(time.RFC3339), Bound: ChannelAsksBound}, true
|
||||
}
|
||||
|
||||
// EndedAtChannel is, of the operator-channel seat's answer of its verb asks, each held ask whose asking ended there
|
||||
// expired or cancelled, as the teller says it in its event ask-ended, so the assigner writes it again by the rules of that
|
||||
// event (AskedAgain), with its compare-and-set: an ended ask said before the assigner's consumer existed is not delivered
|
||||
// to it (issue 1966 note 155, ADR 0668 §4). cut is the answer's own cut, said by the caller; an answer that does not read
|
||||
// is the error.
|
||||
func EndedAtChannel(held map[string][]byte, answer []byte) (ended []tellerevents.Ended, cut string, err error) {
|
||||
var got oc.AsksAnswer // the seat's declared answer: each ask's key and its state, and the cut
|
||||
if err := json.Unmarshal(answer, &got); err != nil {
|
||||
return nil, "", fmt.Errorf("assign: the operator channel's asks do not read: %w", err)
|
||||
}
|
||||
byID := map[string]string{}
|
||||
for key, value := range held {
|
||||
if _, ask, err := DecodeAsk(key, value); err == nil {
|
||||
byID[ask.ID] = key
|
||||
}
|
||||
}
|
||||
for _, v := range got.Asks {
|
||||
if v.State != occ.OutcomeExpired && v.State != occ.OutcomeCancelled {
|
||||
continue
|
||||
}
|
||||
_, id, ok := oc.ParseAsksByAskerIDKey(v.Key)
|
||||
if key, held := byID[id]; ok && held {
|
||||
ended = append(ended, tellerevents.Ended{Key: key, Ask: id, Outcome: v.State})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(ended, func(a, b tellerevents.Ended) int { return strings.Compare(a.Key, b.Key) })
|
||||
return ended, got.Cut, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user