novox/mesh-host
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Milestone 47 G: mesh-host's part of the one release #228

Merged
mesh-tools merged 65 commits from feat/47-G-join12 into main 2026-11-13 13:09:30 +00:00
Conversation 2 Commits 65 Files Changed 108
+7643 -840
108 changed files with 7643 additions and 840 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-compose/main.go
+11 -3
View File
@@ -11,9 +11,12 @@
// checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to
// compose; -declarations answers each machine's declared paths with their owners and modes.
//
// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue
// directory, and reads
// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
// Run as a pipeline step it reads its input, a composer.Request at depth compose, from the file named by the environment
// variable MESH_RUN_INPUT and answers the run's output as its composer.Verdict on the last line of standard output
// (design 47 G A-167).
//
// It opens no network connection, bus, store or file beyond its standard input, the -request file, the file
// MESH_RUN_INPUT names and the -catalogue directory, and reads no other environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error);
// 1 the request cannot be read or run, said on standard error; 2 the flags are wrong.
package main
@@ -65,6 +68,11 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
}
var raw []byte
var err error
// A pipeline step hands its input as a file named by MESH_RUN_INPUT (design 47 G A-167); -request, when given, wins.
runInput := os.Getenv("MESH_RUN_INPUT")
if *requestFile == "" && runInput != "" {
*requestFile = runInput
}
if *requestFile != "" {
raw, err = os.ReadFile(*requestFile)
} else {
cmd/mesh-compose/main_test.go
+29 -4
View File
@@ -3,8 +3,8 @@ package main
import (
"bytes"
"encoding/json"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"go/build"
"os"
@@ -267,6 +267,31 @@ func TestTheRequestIsReadFromAFile(t *testing.T) {
}
}
// A pipeline step names its input file in MESH_RUN_INPUT: the command reads the request there and the verdict is the last
// line of standard output (design 47 G A-167).
func TestThePipelineStepReadsMESH_RUN_INPUT(t *testing.T) {
file := filepath.Join(t.TempDir(), "input.json")
r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{
"demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}}
if err := os.WriteFile(file, encode(t, r), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("MESH_RUN_INPUT", file)
var out, errOut bytes.Buffer
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 0 {
t.Fatalf("exit %d: %s", code, errOut.String())
}
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
var v composer.Verdict
if err := json.Unmarshal([]byte(lines[len(lines)-1]), &v); err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted {
t.Fatalf("last line is no verdict: %v\n%s", err, out.String())
}
t.Setenv("MESH_RUN_INPUT", file+".gone")
if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "cannot be read") {
t.Errorf("a missing input file: exit %d, %q", code, errOut.String())
}
}
// The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error.
func TestTheExitCodesSayWhatFailed(t *testing.T) {
unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}})
@@ -322,7 +347,7 @@ func ownPackages(t *testing.T) map[string][]string {
}
// 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard
// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and
// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types) and catalog key codec (a catalog entry's key, design 47 G A-144), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and
// generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or
// another process.
func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
@@ -334,7 +359,7 @@ func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) {
for path, imports := range own {
for _, imp := range imports {
std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".")
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") {
if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") {
t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp)
}
for _, f := range forbidden {
cmd/mesh-host/applied_store.go
+20
View File
@@ -1,12 +1,16 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/compositions"
)
// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the
@@ -105,3 +109,19 @@ func (s appliedStore) Drop(module string) error {
}
return nil
}
// ownModuleApplied is the module of the composition the store holds at the fingerprint fp, the engine's own build's
// (design 47 G §527, A-149): "" when fp is none or no record holds it.
func ownModuleApplied(s appliedStore, fp string) string {
if fp == "" {
return ""
}
all, _ := s.Load()
for module, raw := range all {
var c compositions.Composed
if module != agent.NodeModule && json.Unmarshal(raw, &c) == nil && c.FP == fp && c.Module != "" {
return c.Module
}
}
return ""
}
cmd/mesh-host/engineseat.go
+37 -2
View File
@@ -18,6 +18,7 @@ import (
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/selfcompose"
@@ -284,7 +285,7 @@ const compositionReadFor = 20 * time.Second
// compositionFrom answers the verb composition from the node agent that runs now (issue 774); refused in words while
// none does.
func compositionFrom(current *agentNow) func(module, digest string, diff bool) (any, error) {
func compositionFrom(current *agentNow, withheld func(ctx context.Context, module string) ([]link.WithheldSubject, string)) func(module, digest string, diff bool) (any, error) {
return func(module, digest string, diff bool) (any, error) {
ag := current.get()
if ag == nil {
@@ -292,7 +293,17 @@ func compositionFrom(current *agentNow) func(module, digest string, diff bool) (
}
ctx, cancel := context.WithTimeout(context.Background(), compositionReadFor)
defer cancel()
return ag.Composition(ctx, module, digest, diff)
view, err := ag.Composition(ctx, module, digest, diff)
if err != nil {
return view, err
}
// And what the grants part withholds of it here, through its verb withheld (design 46 approval mark §6).
subjects, unknown := withheld(ctx, module)
for _, s := range subjects {
view.Withheld = append(view.Withheld, agent.WithheldView{Subject: s.Subject, Why: s.Why})
}
view.WithheldUnknown = unknown
return view, nil
}
}
@@ -390,3 +401,27 @@ func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow
return json.RawMessage(raw), nil
}
}
// firstViewsFrom answers the seat's verb first-views from the composer of this run (design 47 G §698): the views the node-engine
// follows (none), the entries it keeps and the wants it composes from.
func firstViewsFrom(composer *atomic.Pointer[selfcompose.Composer]) func() (any, error) {
return func() (any, error) {
c := composer.Load()
if c == nil {
return nil, errors.New("this node composes none of its own modules now (it applies the controller's compositions, or its node agent is starting), so it keeps no entry")
}
views, entries, wants := c.FirstViews()
return map[string]any{"first-views": views, "entries": entries, "wants": wants}, nil
}
}
// entryAgainFrom asks one entry of the catalog half again, for the seat's verb entry-again (design 47 G §698).
func entryAgainFrom(composer *atomic.Pointer[selfcompose.Composer]) func(ctx context.Context, fingerprint string) error {
return func(ctx context.Context, fingerprint string) error {
c := composer.Load()
if c == nil {
return errors.New("this node composes none of its own modules now, so it keeps no entry to ask again")
}
return c.EntryAgain(ctx, fingerprint)
}
}
cmd/mesh-host/keptartifact.go
+71
View File
@@ -0,0 +1,71 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"github.com/novox/mesh-host/internal/apply"
)
// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the
// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine
// reads them and checks them against the digest. Never another module's files: the seat's verb is the way.
// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object.
var (
keptAskWithin = 30 * time.Second
keptObjectWithin = 10 * time.Minute
)
// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build
// holds the seat of the language it was built in, and the others do not answer.
var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat}
// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node.
func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error),
ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error),
object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch {
return func(ctx context.Context, module, digest string) ([]byte, bool, error) {
node, fp, kept, err := buildOf(ctx, module, digest)
if err != nil || !kept {
return nil, false, err
}
hex := strings.TrimPrefix(digest, "sha256:")
var notHeld []string
for _, seat := range keptBuildSeats {
raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact,
nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin)
if err != nil {
notHeld = append(notHeld, seat+": "+err.Error())
continue
}
var got nodebuildgo.KeptArtifactAnswer
if json.Unmarshal(raw, &got) != nil {
return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node)
}
switch {
case got.Error != "":
return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error)
case got.Refused != "":
return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words)
case got.Store == "" || got.Object == "" || got.Digest != hex:
return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex)
}
octx, cancel := context.WithTimeout(ctx, keptObjectWithin)
body, err := object(octx, got.Store, got.Object, got.Size+1)
cancel()
if err != nil {
return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err)
}
return body, true, nil
}
return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)",
node, module, strings.Join(notHeld, "; "))
}
}
cmd/mesh-host/keptartifact_test.go
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
)
// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47).
func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) {
hex := strings.Repeat("a", 64)
stored := func(string) (json.RawMessage, error) {
return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil
}
for _, row := range []struct {
name string
kept bool
answer func(seat string) (json.RawMessage, error)
object error
wantKept bool
wantErr string
wantSeat string
}{
{"a build kept nothing is not asked", false, nil, nil, false, "", ""},
{"the seat of the language answers", true, func(seat string) (json.RawMessage, error) {
if seat != "node-build-node" {
return nil, errors.New("no responder")
}
return stored(seat)
}, nil, true, "", "node-build-node"},
{"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") },
nil, true, "novox keeps the build of distribution", ""},
{"the seat refuses it", true, func(string) (json.RawMessage, error) {
return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil
}, nil, true, "refuses the kept build of distribution: not kept", ""},
{"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"},
} {
t.Run(row.name, func(t *testing.T) {
var seats []string
fetch := keptArtifactFetch(
func(context.Context, string, string) (string, string, bool, error) {
return "novox", "f1", row.kept, nil
},
func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) {
if node != "novox" || verb != "kept-artifact" {
t.Errorf("asked %s of %s on %s", verb, seat, node)
}
seats = append(seats, seat)
return row.answer(seat)
},
func(_ context.Context, store, name string, limit int64) ([]byte, error) {
if store != "kept-builds" || name != hex || limit < 5 {
t.Errorf("read %s/%s up to %d", store, name, limit)
}
return []byte("bytes"), row.object
})
body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex)
if kept != row.wantKept {
t.Fatalf("kept = %v", kept)
}
if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) {
t.Fatalf("got %q, %v", body, err)
}
if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) {
t.Fatalf("the error is %v, want %q", err, row.wantErr)
}
if !row.kept && len(seats) > 0 {
t.Errorf("a build that kept nothing asked %v", seats)
}
})
}
}
cmd/mesh-host/logins.go
+7 -6
View File
@@ -87,15 +87,16 @@ func followLogins(ctx context.Context, session *link.Session, node string, hear
}
}
// loginSecretModule is the module the vault names this engine's login values under: own/<node>/mesh-host/<login> (ADR 0583 §3,
// insight).
const loginSecretModule = "mesh-host"
// fetchLogin asks the vault, through its existing hand-out of a machine's own secrets, for the value of login, sealed to this
// node's key, and opens it with the sealing key; no new verb is asked for it.
func fetchLogin(queue *link.Session, statePath string) func(context.Context, string) (string, error) {
unseal := sealOpener(statePath)
return func(ctx context.Context, login string) (string, error) {
// The vault names the login values under the engine's own module, which is §527's (§634).
own, err := queue.Own.Wait(ctx)
if err != nil {
return "", fmt.Errorf("the node-engine's own module is not named yet, so the vault's name of its login value is not known: %w", err)
}
raw, err := queue.AskVault(ctx, "own", map[string]any{})
if err != nil {
return "", err
@@ -111,7 +112,7 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str
return "", fmt.Errorf("the vault's own answer is not one: %w", err)
}
for _, o := range a.Own {
if o.Module != loginSecretModule || o.Name != login {
if o.Module != own || o.Name != login {
continue
}
sealed := string(o.Sealed)
@@ -125,7 +126,7 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str
}
return string(value), nil
}
return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), loginSecretModule, login)
return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), own, login)
}
}
cmd/mesh-host/main.go
+31 -2
View File
@@ -16,6 +16,7 @@ import (
"errors"
"flag"
"fmt"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
"io"
"net"
@@ -35,6 +36,7 @@ import (
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/bundle"
"github.com/novox/mesh-host/internal/compositions"
"github.com/novox/mesh-host/internal/firewall"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/inventory"
@@ -42,6 +44,7 @@ import (
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/network"
"github.com/novox/mesh-host/internal/outward"
"github.com/novox/mesh-host/internal/processbuilds"
"github.com/novox/mesh-host/internal/profile"
"github.com/novox/mesh-host/internal/reachable"
"github.com/novox/mesh-host/internal/store"
@@ -985,7 +988,12 @@ func runLink(ctx context.Context, opts options) error {
// What this engine trusts a mesh signature from: the vault's key alone, once read from its signers (novox/hq ADR 0550
// §2; issue 1126: the controller's key never, the controller ended by its unassignment).
trust := newMeshTrust()
own := link.NewOwnName()
// The engine's own module, first from its applied record of the build whose fingerprint is its own (design 47 G §527,
// §634, A-149); on a first start with no such record, from its first composing.
own.Set(ownModuleApplied(appliedStoreFor(opts.state), os.Getenv(compositions.FingerprintVar)))
queue := &link.Session{
Own: own,
Membership: membership,
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
Keys: recordKeys(opts.state, trust.Trusts),
@@ -1014,10 +1022,23 @@ func runLink(ctx context.Context, opts options) error {
queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch)
go reconcileActs(ctx, queue.Acts)
current := &agentNow{}
queue.Acts.Composition = compositionFrom(current)
queue.Acts.Composition = compositionFrom(current, queue.WithheldOf)
queue.Acts.Compose = composeFrom(current, &composerNow)
queue.Acts.Packages = packagesFrom(mine.Node, queue, &composerNow)
queue.Acts.FirstViews, queue.Acts.EntryAgain = firstViewsFrom(&composerNow), entryAgainFrom(&composerNow)
queue.Acts.SeatList, queue.Acts.SeatListAgain = seatListRead, seatListAgain
queue.Acts.ProcessBuilds = func(module string) (any, error) {
d, err := declaredNow(opts.state, nil)
if errors.Is(err, store.ErrNothingDeclared) {
d, err = nil, nil
}
if err != nil {
return nodeengine.ProcessBuilds{Node: mine.Node, Error: "what runs on this node does not read: " + err.Error()}, nil
}
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
defer cancel()
return processbuilds.Reader{Node: mine.Node, Run: processbuilds.Runner(opts.run())}.Read(ctx, d, module), nil
}
queue.Acts.ComposeCandidate = composeCandidateFrom(mine.Node, &shadowNow)
setRunsHere(runsFromAgent(current))
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
@@ -1381,7 +1402,15 @@ func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health,
if c := machinesNow.Load(); c != nil {
machines.ByAddress, machines.Hub = c.Machines()
}
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines))
report := link.NodeHealthOf(queue.Membership.Node, h, peers, machines)
if queue.Fallback != nil && report.Fallback == nil {
report.Fallback = queue.Fallback() // node-account-fallback (ADR 0583 §6), as the written report carries it
}
_, err := queue.WriteNodeHealth(ctx, report)
// The findings the report says are raised by the engine here, at every look, written to the bus or not (design 47 G A-197).
if keeper := ownFindingsNow.Load(); keeper != nil {
keeper.Report(report, machines.ByAddress)
}
switch {
case err != nil && err.Error() != *unwritten:
*unwritten = err.Error()
cmd/mesh-host/nodeagent.go
+135 -9
View File
@@ -25,8 +25,10 @@ import (
"github.com/novox/mesh-host/internal/compose/frombus"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/seatwait"
"github.com/novox/mesh-host/internal/selfcompose"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
@@ -88,7 +90,7 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
return agent.Have{}, false
}
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true
Sealed: got.SealedAt, Wait: got.Wait}, true
}
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
@@ -106,7 +108,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b
return agent.Have{}, false, err
}
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true, nil
Sealed: got.SealedAt, Wait: got.Wait}, true, nil
}
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
@@ -123,7 +125,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
}
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
Failed: phase == string(nodeengineclient.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait})
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait, Wait: x.Wait})
}
func (h sessionHaves) Removed(ctx context.Context, module string) {
@@ -447,6 +449,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
began := time.Now()
node := session.Membership.Node
var keptFetch atomic.Pointer[apply.KeptFetch] // the composer's, set once it exists
composedBucket, err := compositions.BucketFor(node)
if err != nil {
cancel()
@@ -549,6 +552,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
Backoff: link.Backoff,
Apply: func(ctx context.Context, c compositions.Composed, others []compositions.Composed) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
if kf := keptFetch.Load(); kf != nil {
actx = apply.WithKept(actx, *kf)
}
// While the grants part's list is followed, the controller's list in the bus module's composition is not
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
held, herr := users.Hold(c)
@@ -573,6 +579,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
sayFacts(ctx, ag)
return res
},
Stop: func(ctx context.Context, module string) agent.Result {
return stopComposition(ctx, opts, module)
},
Remove: func(ctx context.Context, module string, others []compositions.Composed) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
@@ -591,14 +600,33 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
own := &agentNow{}
// And the seat list it reads through the seat mesh-catalog, from its bootstrapped definitions until it has: the
// composer hands nothing new meanwhile, and the node says it has waited five minutes (novox/hq ADR 0636 §1).
seats, err := startSeatList(run, session, node, say)
// The finding writer is started first, so it holds what an earlier run left before the reader writes or clears any.
if session.Own == nil {
session.Own = link.NewOwnName()
}
findings := &lateFindings{own: session.Own, make: func(module string) (*findingsOfTheNode, error) {
return newFindingsOfTheNode(session, node, module, say)
}}
findings.Start(run, say)
seats, err := startSeatList(run, session, node, findings, say)
if err != nil {
// No reader is no gate that lets everything through: the node acts on nothing new, as one that waits does.
say(err.Error() + "; the node composes nothing new until it is started again and reads the list (novox/hq ADR 0636 §1)")
}
// The marks of the haves run off the composition pass, each bounded (see marker).
marks := newMarker(run)
heldFindings := &seatwait.HeldKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say}
// And what only this engine knows of itself, raised where it knows it (design 47 G A-197): its haves' failed phase, health
// and kept words as they are written, and each part of its health report at every look.
ownFindings := &seatwait.NodeKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say}
session.SetHaveSaid(ownFindings.Have, ownFindings.HaveGone)
ownFindingsNow.Store(ownFindings)
hooks := composerHooks{
entryNotFound: func(fp string, notFound bool) {
if err := seatwait.SayEntryNotFound(run, findings, node, findings.observer(), fp, notFound, time.Now); err != nil {
say("the finding that the catalog half holds no entry of " + fp + " cannot be kept: " + err.Error())
}
},
seatList: seats,
seatListFailed: err != nil,
running: func() []compositions.Composed {
@@ -626,6 +654,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
// A module held alone marks that module, never the others (issue 1126); the marking never holds the pass.
marks.set("refused", func(ctx context.Context) { now.NotComposedEach(ctx, why, held) })
}
// And each module held says it as the finding `module-held`, cleared when it is composed again or no longer
// wanted here. A node refused whole (why) says nothing of its modules: their findings stand as they were.
if why == "" && run.Err() == nil {
marks.set("module-held", func(context.Context) { heldFindings.Reconcile(held) })
}
},
// And what it keeps of a module as it runs, for a manifest that does not say it yet: on the module's have (ADR 0601 §4).
kept: func(kept map[string][]string) {
@@ -656,20 +689,35 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
// composition bucket) is never applied. While the seat's assignment holds no key for this node it composes
// nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again.
composer := selfComposer(session, node, hooks, say)
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
composer.AlsoEntryAgain = shadow.EntryAgain
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
go followRunning(run, session, composer, say)
go followBuildEnds(run, session, composer, say)
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday
env.Recovery = recoveryEnv(session, composer)
env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting
keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject)
keptFetch.Store(&keptFetchFn)
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
ag = agent.New(env)
own.set(ag)
current.set(ag)
// The node's own record is stated before the composer starts, from what the node knows of itself and with no bus (A-120).
sayFacts(run, nil)
err = ag.Run(run)
stopRun()
cancel()
current.set(nil)
composerNow.Store(nil)
ownFindingsNow.Store(nil)
session.SetHaveSaid(nil, nil)
session.SetManifestOf(nil)
apply.UseEngineModule(nil)
shadowNow.Store(nil)
if ctx.Err() != nil {
return
@@ -828,6 +876,30 @@ func removeComposition(ctx context.Context, opts options, module string, others
return agent.Result{}
}
// stopComposition stops a module's units and containers from the node's record and removes nothing (design 47 G A-98,
// §297): what the module kept stays, so a revert restores it before the reverted build starts.
func stopComposition(ctx context.Context, opts options, module string) agent.Result {
applying.Lock()
defer applying.Unlock()
unlock, err := store.Lock(opts.state, nil)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
defer unlock()
known, err := store.Load(opts.state)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
built, err := system.For(builtFor)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
if _, err := apply.StopModule(ctx, built, known, module, opts.run()); err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
return agent.Result{}
}
// busyModules are the modules whose last apply left a container running because it said busy.
func busyModules() []string {
busyMu.Lock()
@@ -846,6 +918,10 @@ func busyModules() []string {
// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target.
var machinesNow atomic.Pointer[selfcompose.Composer]
// ownFindingsNow is the keeper of the findings this engine raises of itself in the run that composes the node's own now, for the
// health loop (writeHealthReport); nil while no run holds one.
var ownFindingsNow atomic.Pointer[seatwait.NodeKeeper]
// composerNow is the composer of the run that composes the node's own now, nil while the node applies the controller's
// compositions or no run is going: what the verb compose asks a pass of (issue 774).
var composerNow atomic.Pointer[selfcompose.Composer]
@@ -1053,7 +1129,9 @@ type composerHooks struct {
unseal func(string) ([]byte, error)
refused func(why string, held map[string]string)
kept func(kept map[string][]string)
held func(module string) (compositions.Seals, []string, error)
// entryNotFound is selfcompose.Env's EntryNotFound.
entryNotFound func(fp string, notFound bool)
held func(module string) (compositions.Seals, []string, error)
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
vaultAuthority func() string
certify func(ctx context.Context) error
@@ -1076,13 +1154,15 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f
// And every composition it makes in its composition state composed-<node>, a record the provisioner, the assigner
// and the view read and nothing applies (novox/hq ADR 0593 §6).
env.ComposedState = link.EngineComposed{Session: session, Node: node}
// And each composition it writes there said as the seat's event composed, which the assigner waits on.
env.SayComposed = session.SayComposed
return selfcompose.New(env)
}
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
// to assignment candidates (selfcompose.Shadow) alike.
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
env := selfcompose.Env{Node: node,
env := selfcompose.Env{Node: node, NodeRecord: session.OwnRecord,
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
if bucket == buckets.Assignments || bucket == buckets.ProposalCandidates {
// The seat's states are made when their holder starts: a node-engine that starts first waits for the
@@ -1110,12 +1190,58 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
Hear: session.Hear,
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
Fetch: apply.Fetch,
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept,
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, EntryNotFound: hooks.entryNotFound,
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
if hooks.seatList != nil {
env.SeatList, env.SeatListChanged = hooks.seatList.reader, hooks.seatList.changed
} else if hooks.seatListFailed {
env.SeatList = selfcompose.Unread{}
}
// And the shadow too: the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's
// verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read.
env.Entry = selfcompose.EntryCall(session.AskMeshSeat)
return env
}
// followRunning keeps the composer told what every node runs, from the seat node-engine's event running, for the run it is
// in: the stream or the link not being there is said and tried again after a backoff, never polled.
func followRunning(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) {
keepFollowing(run, "this node does not hear what the nodes run, and tries again: ", func(ctx context.Context) error {
return session.FollowRunning(ctx, composer.NoteRunning, composer.RunningRead)
}, composer.RunningLost, say)
}
// followBuildEnds keeps the composer told how the builder's builds ended, from its event build.ended, for the run it is in
// (design 47 G A-293): a placed module's newest build that ended failed is said as a refusal in the composition state.
func followBuildEnds(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) {
keepFollowing(run, "this node does not hear how the builder's builds end, so a failed build is not said as refused, and it tries again: ",
func(ctx context.Context) error {
return session.FollowBuildEnds(ctx, composer.NoteBuildEnded, composer.BuildEndsRead)
}, composer.BuildEndsLost, say)
}
// keepFollowing follows a stream with follow for the run, lost called each time it ends; why it ended is said once until it
// changes, and it is followed again after a backoff, never polled.
func keepFollowing(run context.Context, line string, follow func(context.Context) error, lost func(), say func(string)) {
said := ""
for try := 1; run.Err() == nil; try++ {
began := time.Now()
err := follow(run)
lost()
if run.Err() != nil {
return
}
if time.Since(began) > time.Minute {
try = 1
}
if why := fmt.Sprint(err); why != said {
say(line + why)
said = why
}
select {
case <-run.Done():
return
case <-time.After(link.Backoff(try)):
}
}
}
cmd/mesh-host/nodeagent_integration_test.go
+139 -38
View File
@@ -11,11 +11,11 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
builderbuilds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
engcomp "github.com/novox/mesh-host/internal/compositions"
@@ -78,6 +78,55 @@ type itBed struct {
// holder re-serves (issue 1966).
seatAway atomic.Bool
findings nats.KeyValue
// entries are the entries the stand-in of the seat mesh-catalog answers its verb entry from, by fingerprint (design 47 G §421).
entries map[string]meshcatalog.CatalogEntry
}
// registerEntry makes the stand-in answer the entry of fp: module's manifest and its artifacts (refs `<module>:<name>`).
func (b *itBed) registerEntry(module, fp, manifest string, artifacts ...string) {
b.t.Helper()
var m map[string]any
if err := json.Unmarshal([]byte(manifest), &m); err != nil {
b.t.Fatal(err)
}
e := meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, ManifestDigest: meshcatalogclient.ManifestDigest([]byte(manifest)),
Repository: "novox/" + module, Commit: strings.Repeat("1", 40)}
for _, a := range artifacts {
e.Artifacts = append(e.Artifacts, meshcatalog.Artifact{Ref: module + ":" + a, Kind: "archive", Language: "go"})
}
b.mu.Lock()
b.entries[fp] = e
b.mu.Unlock()
}
// running says the seat node-engine's event running of module on node at fp, as that node's engine does.
func (b *itBed) running(node, module, fp, manifestDigest string) {
b.t.Helper()
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: module, Fingerprint: fp, Manifest: manifestDigest, At: time.Now().UTC()})
if _, err := b.js.Publish(nodeengine.SubjectRunning(node, module), raw); err != nil {
b.t.Fatal(err)
}
}
// standInEntry answers the seat mesh-catalog's verb entry from the registered entries, and as the holder refuses an unknown one.
func (b *itBed) standInEntry() {
sub, err := b.conn.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) {
var in meshcatalog.EntryInput
_ = json.Unmarshal(m.Data, &in)
b.mu.Lock()
e, ok := b.entries[in.Fingerprint]
b.mu.Unlock()
if !ok {
_ = m.Respond([]byte(`{"error":"no entry of fingerprint ` + in.Fingerprint + ` is kept"}`))
return
}
out, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: e}})
_ = m.Respond(out)
})
if err != nil {
b.t.Fatal(err)
}
b.t.Cleanup(func() { _ = sub.Unsubscribe() })
}
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
@@ -130,7 +179,7 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed {
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
b := &itBed{t: t, dir: t.TempDir()}
b := &itBed{t: t, dir: t.TempDir(), entries: map[string]meshcatalog.CatalogEntry{}}
b.conn, b.nodeConn = aServer(t, perms, func(err error) {
b.mu.Lock()
b.refusals = append(b.refusals, err.Error())
@@ -159,7 +208,20 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed {
if b.findings, err = b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: link.FindingsBucket, History: 1}); err != nil {
t.Fatal(err)
}
// The engine's own finding stream and the keeper's ask stream (design 47 G §383, §390), made as the bus module makes them.
for _, c := range []nats.StreamConfig{{Name: "MESH-HOST-FINDINGS", Subjects: []string{"mesh.stream.mesh-host.event.>"}},
{Name: condseat.StreamFindingsAsked, Subjects: []string{condseat.SubjectFindingsAsked}}} {
if _, err = b.js.AddStream(&c); err != nil {
t.Fatal(err)
}
}
// The stream the seat node-engine's event running is kept in, the last of each subject (design 47 G §199).
if _, err = b.js.AddStream(&nats.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll},
MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
b.standInSeat()
b.standInEntry()
b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second}
if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil {
t.Fatal(err)
@@ -168,10 +230,19 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed {
return b
}
// itOwnModule is the node-engine's own module of the bed: what its applied record of its own build says (§527, §634).
const itOwnModule = "mesh-host"
func itOwn() *link.OwnName {
o := link.NewOwnName()
o.Set(itOwnModule)
return o
}
// start runs the node agent under a session on the bus.
func (b *itBed) start() (stop func()) {
b.t.Helper()
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
ctx, cancel := context.WithCancel(context.Background())
@@ -419,9 +490,8 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
}
return kv
}
assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests)
assignment, manifests := kvOf(buckets.Assignments), kvOf(buckets.Manifests)
resolved := kvOf(selfcompose.ResolvedBucket)
builds := kvOf(builderbuilds.BuilderBuildsBucket)
candidates := kvOf(buckets.ProposalCandidates)
packages := kvOf(link.PackagesBucket)
path := filepath.Join(b.dir, "web.conf")
@@ -435,13 +505,11 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
`{"id":"salut","type":"file","path":"` + more("salut") + `","mode":"0644","content":"salut\n","if-config":["greeting=salut"]},` +
`{"id":"bad","type":"file","path":"` + path + `/x","mode":"0644","content":"x\n","if-config":["greeting=boom"]},` +
`{"id":"after","type":"file","path":"` + more("after") + `","mode":"0644","content":"after\n","if-config":["greeting=boom"]},` +
`{"id":"tools","type":"package","package":"web-tools"}],` +
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
`{"id":"tools","type":"package","package":"web-tools"}]}`
digest := meshcatalogclient.ManifestDigest([]byte(man))
mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man})
mkey, _ := meshcatalogclient.ManifestsKey(digest)
fp := fpIt("a")
modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
place, _ := meshmoduleassignerclient.EncodeAssignments(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}})
rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}})
// it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both.
@@ -455,30 +523,29 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
mdigest := meshcatalogclient.ManifestDigest([]byte(mailer))
mlraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mailer", Manifest: mailer})
mlkey, _ := meshcatalogclient.ManifestsKey(mdigest)
mlmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
places2, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: "it2", Modules: []meshmoduleassigner.PlacedModule{{Module: "mailer"}}})
rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
pkey2, _ := meshmoduleassignerclient.NodeKey("it2")
hkey2, _ := nec.HaveNodeKey("it2")
buildKey, _ := builderbuilds.BuilderBuildKey("web", fp)
done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest,
Artifacts: []builderbuilds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}})
firstBuild := done
if checkBuild {
firstBuild = []byte(`{"v":1,"state":"no such state"}`)
}
for _, w := range []struct {
kv nats.KeyValue
key string
value []byte
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
}{{manifests, mkey, mraw}, {b.haves, hkey, rec},
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)},
{manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]},
{resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}, {builds, buildKey, firstBuild}} {
{manifests, mlkey, mlraw}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]},
{resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}} {
if _, err := w.kv.Put(w.key, w.value); err != nil {
t.Fatal(err)
}
}
// Design 47 G §421: the node reads the entry its want names, through the seat mesh-catalog's verb entry, and the part's digest
// from the want; what it hears another node run (the mailer on it2) is read by the same fingerprint.
if !checkBuild {
b.registerEntry("web", fp, man)
}
b.registerEntry("mailer", fpIt("b"), mailer)
b.running("it2", "mailer", fpIt("b"), mdigest)
want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true})
wkey := meshwants.WantsByWantNodeModuleKey(itNode, "web")
if _, err := b.wants.Put(wkey, want); err != nil {
@@ -499,17 +566,18 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
t.Fatal(err)
}
if checkBuild {
// The build of the placed module is read through the node's session: one that does not decode is said, and nothing
// is applied; a done one, then a config event, and the node composes.
within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey)
// The entry of the build its want names is asked of the catalog half through the node's session: one the half holds none
// of holds the want (entry-missing, naming the fingerprint) and nothing is applied; once the half holds it and it is
// asked again by hand, then a config event, and the node composes.
within(t, 5*time.Second, "the have does not say the entry of web's build is missing", func() bool {
h := b.have("web")
return h.Phase == nec.PhaseHeld && strings.HasPrefix(h.Detail, agent.EntryMissing) && strings.Contains(h.Detail, fp)
})
if b.file("web") != "" {
t.Fatal("composed with a build key that does not decode")
t.Fatal("composed with an entry the catalog half does not hold")
}
if _, err := builds.Put(buildKey, done); err != nil {
b.registerEntry("web", fp, man)
if err := composerNow.Load().EntryAgain(context.Background(), fp); err != nil {
t.Fatal(err)
}
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil {
@@ -784,7 +852,10 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test
}
folded, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` +
`{"id":"nats.server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: old\n"},` +
`{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `"}]}`))
`{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `",` +
// As the node-engine starts every container since the fingerprint handover (47/G W13.1): the fingerprint of the
// want it applies, here the first want's, so that want changes the server's spec in nothing.
`"env":{"MESH_FINGERPRINT":"` + fpIt("e") + `"}}]}`))
if err != nil {
t.Fatal(err)
}
@@ -813,7 +884,6 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test
}
assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests)
kvOf(selfcompose.ResolvedBucket)
builds := kvOf(builderbuilds.BuilderBuildsBucket)
kvOf(buckets.ProposalCandidates)
kvOf(link.PackagesBucket)
// The bus module at a build, done: its manifest written and the indexer's module naming it at fp.
@@ -826,14 +896,10 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test
mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "nats", Manifest: man})
mkey, _ := meshcatalogclient.ManifestsKey(digest)
modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)})
b.registerEntry("nats", fp, man) // the node reads the entry its want names (design 47 G §421)
if _, err := manifests.Put(mkey, mraw); err != nil {
t.Fatal(err)
}
bkey, _ := builderbuilds.BuilderBuildKey("nats", fp)
done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest})
if _, err := builds.Put(bkey, done); err != nil {
t.Fatal(err)
}
if _, err := modules.Put("nats", modv); err != nil {
t.Fatal(err)
}
@@ -1019,6 +1085,11 @@ func (c *fiveMinutes) pass() {
}
}
// findingSubject is the subject of one of the engine's finding events on node (views.FindingSubjects).
func findingSubject(event, node string) string {
return "mesh.stream." + itOwnModule + ".event." + event + "." + node
}
// ADR 0636 §1, §8, whole against a real bus and the stand-in of the seat: while the seat refuses, the node composes and
// applies nothing new, and keeps running what it ran; after the bound it writes one finding in its state findings; when
// the seat names its list the module is applied and the finding is cleared; a node started again with the seat gone
@@ -1072,6 +1143,27 @@ func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) {
if b.file("web") != "" {
t.Fatalf("the node applied web with its finding written: %q", b.file("web"))
}
// Design 47 G §383, §384, §390 (test 107): the key is followed by the event on the engine's own stream, the engine stated
// its findings at its start, and an ask of the keeper is answered by a new statement.
lastEvent := func(event string) (string, bool) {
m, err := b.js.GetLastMsg("MESH-HOST-FINDINGS", findingSubject(event, itNode))
if err != nil {
return "", false
}
return string(m.Data), true
}
within(t, 5*time.Second, "the node did not say finding-written after its key", func() bool {
got, ok := lastEvent("finding-written")
return ok && strings.Contains(got, `"kind":"seat-list-unread"`) && strings.Contains(got, `"observer":"mesh-host"`)
})
within(t, 5*time.Second, "the node did not state its findings at its start", func() bool { _, ok := lastEvent("findings-stated"); return ok })
if _, err := b.js.Publish(condseat.SubjectFindingsAsked, []byte(`{"ask":"a1","at":"2026-11-11T12:00:00Z","by":"test"}`)); err != nil {
t.Fatal(err)
}
within(t, 5*time.Second, "the node did not answer the keeper's ask with a statement naming the finding", func() bool {
got, ok := lastEvent("findings-stated")
return ok && strings.Contains(got, `"asked":"a1"`) && strings.Contains(got, `"kind":"seat-list-unread"`)
})
b.seatSilent.Store(false)
// The hand act of the operator and the healers (the verb seat-list-again): a read now, as the seat's event would make.
@@ -1083,6 +1175,15 @@ func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) {
_, err := b.findings.Get(finding)
return errors.Is(err, nats.ErrKeyNotFound)
})
within(t, 5*time.Second, "the node did not say finding-cleared for the seat list", func() bool {
for i := uint64(1); i < 40; i++ {
if m, err := b.js.GetMsg("MESH-HOST-FINDINGS", i); err == nil && m.Subject == findingSubject("finding-cleared", itNode) &&
strings.Contains(string(m.Data), `"kind":"seat-list-unread"`) {
return true
}
}
return false
})
// Started again with the seat refusing: nothing new is composed, and what ran is neither removed nor changed.
stop()
cmd/mesh-host/ownname_test.go
+77
View File
@@ -0,0 +1,77 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"github.com/novox/mesh-host/internal/link"
)
// Design 47 G §527, §634, A-149: the node-engine's own module is the module of its applied record at its own fingerprint,
// and its finding writer is made, observing as that module, once the name is known: at once from the record, else when the
// first composing names it. No name is fixed in code.
func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testing.T) {
fp := "src2:" + strings.Repeat("a", 64)
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
for module, f := range map[string]string{"node": "", "the-engine": fp, "other": "src2:" + strings.Repeat("b", 64)} {
if err := applied.Keep(module, []byte(`{"module":"`+module+`","fp":"`+f+`"}`)); err != nil {
t.Fatal(err)
}
}
for _, tc := range []struct{ fp, want string }{{fp, "the-engine"}, {"", ""}, {"src2:" + strings.Repeat("c", 64), ""}} {
if got := ownModuleApplied(applied, tc.fp); got != tc.want {
t.Errorf("the own module at %q is %q, want %q", tc.fp, got, tc.want)
}
}
own := link.NewOwnName()
made := make(chan string, 1)
late := &lateFindings{own: own, make: func(module string) (*findingsOfTheNode, error) {
made <- module
return nil, context.Canceled // the writer itself is the SDK's: only when it is made is tested here
}}
ctx, stop := context.WithCancel(context.Background())
defer stop()
var said []string
late.Start(ctx, func(s string) { said = append(said, s) })
if err := late.Write(ctx, condseat.Finding{}); err == nil || late.Held() != nil {
t.Fatalf("a finding was taken before the engine's own module was known: %v", err)
}
if len(made) != 0 {
t.Fatal("the writer was made before a name")
}
own.Set("the-engine") // the first composing names it
select {
case m := <-made:
if m != "the-engine" || late.observer() != "the-engine" {
t.Fatalf("the writer was made as %q, observing as %q", m, late.observer())
}
case <-time.After(5 * time.Second):
t.Fatal("the writer was never made after the name was known")
}
}
// §633: the engine's bound for the first seat list is the file the manifest's `view-waits-minutes` feeds, five minutes
// when it holds nothing usable.
func TestTheViewWaitsBoundIsTheManifestsSetting(t *testing.T) {
was := viewWaitsFile
defer func() { viewWaitsFile = was }()
for content, want := range map[string]time.Duration{"2\n": 2 * time.Minute, "": 5 * time.Minute, "0": 5 * time.Minute, "x": 5 * time.Minute} {
viewWaitsFile = filepath.Join(t.TempDir(), "w")
if err := os.WriteFile(viewWaitsFile, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
if got := viewWaitsBound(); got != want {
t.Errorf("the file %q gives %v, want %v", content, got, want)
}
}
viewWaitsFile = filepath.Join(t.TempDir(), "missing")
if got := viewWaitsBound(); got != 5*time.Minute {
t.Errorf("no file gives %v", got)
}
}
cmd/mesh-host/recovery.go
+89
View File
@@ -0,0 +1,89 @@
package main
import (
"context"
"encoding/json"
"os"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/selfcompose"
)
// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296,
// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the
// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool.
func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery {
ask := func(ctx context.Context, verb string, in, out any) error {
raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in)
if err != nil {
return err
}
return json.Unmarshal(raw, out)
}
return &agent.Recovery{
FormChanges: composer.FormChanges,
SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a)
},
RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a)
},
KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a)
},
Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) {
heard, stop, err := session.HearRecoveryEnding(kind, module, kept)
if err != nil {
return nil, nil, err
}
out := make(chan agent.Ending, 64)
done := make(chan struct{})
go func() {
for {
select {
case ev := <-heard:
select {
case out <- agent.Ending(ev):
case <-done:
return
}
case <-done:
return
}
}
}()
return out, func() { stop(); close(done) }, nil
},
Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) {
return session.KeepBound(ctx, what, id, module, since, d)
},
CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error {
return session.CallTool(ctx, module, c.Tool, c.Arguments)
},
WaitMinutes: recoveryWaitMinutes(),
}
}
// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the
// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71).
var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes"
// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the
// file is missing, empty or not a positive number (design 47 G §554, A-112).
func recoveryWaitMinutes() int {
raw, err := os.ReadFile(recoveryWaitFile)
if err != nil {
return agent.RecoveryWaitMinutes
}
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil || n <= 0 {
return agent.RecoveryWaitMinutes
}
return n
}
cmd/mesh-host/recovery_integration_test.go
+301
View File
@@ -0,0 +1,301 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/nodegrant"
"github.com/novox/mesh-host/internal/selfcompose"
"strings"
"sync"
"sync/atomic"
)
// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears
// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's
// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and
// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant.
func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) {
perms := nodegrant.For(itNode)
perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode,
"mesh.mod.db.tool.rebuild."+itNode)
perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>")
b := newItBedAs(t, perms)
ctx := context.Background()
for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll},
{meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} {
if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
}
js, err := jetstream.New(b.conn)
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil {
t.Fatal(err)
}
say := func(act, outcome string) {
raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()})
if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil {
t.Fatal(err)
}
}
var restoreAsked meshrecovery.RestoreKeptInput
// The kept points the stand-in answers: the one of the first checks, then (below) what the agent's revert finds, and none.
var pointsMu sync.Mutex
points := []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1",
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}
// The restore the agent's revert asks waits for the stand-in to say it restored: gated until released.
var gated atomic.Bool
release := make(chan struct{})
var restores atomic.Int32
for verb, answer := range map[string]func(m *nats.Msg) any{
meshrecovery.VerbKeptPoints: func(*nats.Msg) any {
pointsMu.Lock()
defer pointsMu.Unlock()
return meshrecovery.KeptPointsAnswer{Points: points}
},
meshrecovery.VerbRestoreKept: func(m *nats.Msg) any {
pointsMu.Lock()
_ = json.Unmarshal(m.Data, &restoreAsked)
pointsMu.Unlock()
act := fmt.Sprintf("r%d", restores.Add(1))
if gated.Load() {
go func() { <-release; say(act, meshrecovery.RestoredEventOutcomeDone) }()
} else {
say(act, meshrecovery.RestoredEventOutcomeDone)
}
return meshrecovery.RestoreKeptAnswer{Accepted: act}
},
} {
answer := answer
sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) {
out, _ := json.Marshal(map[string]any{"result": answer(m)})
_ = m.Respond(out)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
var toolArgs map[string]any
var toolCalls atomic.Int32
toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) {
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
pointsMu.Lock()
toolArgs = args
pointsMu.Unlock()
toolCalls.Add(1)
_ = m.Respond([]byte(`{"result":{}}`))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = toolSub.Unsubscribe() })
// A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number.
for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} {
recoveryWaitFile = filepath.Join(t.TempDir(), "w")
if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
if got := recoveryWaitMinutes(); got != want {
t.Errorf("the file %q gives %d minutes, want %d", content, got, want)
}
}
recoveryWaitFile = filepath.Join(t.TempDir(), "missing")
if got := recoveryWaitMinutes(); got != 90 {
t.Errorf("no file gives %d minutes, want 90", got)
}
say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject
session := b.startSession()
r := recoveryEnv(session, nil)
events, stop, err := r.Hear(ctx, "restored", "db", "cache")
if err != nil {
t.Fatal(err)
}
defer stop()
if first := <-events; first.Act != "old" || first.Outcome != "failed" {
t.Fatalf("the first event heard is %+v, want the last one said before", first)
}
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"})
if err != nil || len(pts.Points) != 1 {
t.Fatalf("kept-points: %+v, %v", pts, err)
}
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"})
pointsMu.Lock()
firstAsked := restoreAsked
pointsMu.Unlock()
if err != nil || got.Accepted != "r1" || firstAsked.Snapshot != "snap-1" {
t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, firstAsked)
}
if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" {
t.Fatalf("the ending heard is %+v", ev)
}
err = r.CatchUp(ctx, "db", *pts.Points[0].CatchUp)
pointsMu.Lock()
gotArgs := toolArgs
pointsMu.Unlock()
if err != nil || gotArgs["all"] != true {
t.Fatalf("catch-up: %v, arguments %v", err, gotArgs)
}
expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second)
if err != nil {
t.Fatal(err)
}
defer end()
select {
case <-expired:
case <-time.After(10 * time.Second):
t.Fatal("the bound's end, the bus's marker, was not heard")
}
// Design 47 G §296, §297, §554, §560, A-113: the node agent whole drives a revert want of db through the same recovery wiring
// (recoveryEnv over the session and the node's composer): db runs at a build whose cache is of form 2, and its want goes back to a
// build of form 1. With no point of form 1 taken before the newer build, its have says the restore point is missing, naming it,
// and the newer build runs again. With the point, the have is held (backing up) until the stand-in says restored, then the
// older build is applied and the point's catch-up tool is called.
dbManifest := func(form int, content string) string {
return `{"module":"db","version":"1","state":[{"name":"cache","form":` + fmt.Sprint(form) + `}],"resources":[{"id":"conf","type":"file","path":"` +
filepath.Join(b.dir, "db.conf") + `","mode":"0644","content":"` + content + `\n"}]}`
}
fpOld, fpNew := fpIt("a"), fpIt("b")
b.registerEntry("db", fpOld, dbManifest(1, "v1"))
b.registerEntry("db", fpNew, dbManifest(2, "v2"))
for _, name := range []string{buckets.Manifests, selfcompose.ResolvedBucket, buckets.ProposalCandidates, link.PackagesBucket} {
if _, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 5}); err != nil {
t.Fatal(err)
}
}
assignment, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: buckets.Assignments, History: 5})
if err != nil {
t.Fatal(err)
}
places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "db"}}})
hkey, _ := nec.HaveNodeKey(itNode)
rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}})
if _, err := b.haves.Put(hkey, rec); err != nil {
t.Fatal(err)
}
for key, value := range places {
if _, err := assignment.Put(key, value); err != nil {
t.Fatal(err)
}
}
b.want("db", fpNew, "", "")
stopAgent := b.start()
defer stopAgent()
within(t, 20*time.Second, "db does not run its newer build", func() bool {
h := b.have("db")
return h.Fp == fpNew && h.Phase == nec.PhaseRunning && b.file("db") == "v2\n"
})
// The engine says what runs, as it does after each apply: the composer keeps the entry of the running build while the want moves on.
b.running(itNode, "db", fpNew, meshcatalogclient.ManifestDigest([]byte(dbManifest(2, "v2"))))
// The stand-in holds no point of form 1: the revert is held, the have names the point missing, the newer build runs.
pointsMu.Lock()
points = nil
pointsMu.Unlock()
b.want("db", fpOld, "", "")
within(t, 20*time.Second, "the have does not say the restore point is missing", func() bool {
h := b.have("db")
return h.Phase == nec.PhaseHeld && strings.Contains(h.Detail, "restore point missing") &&
strings.Contains(h.Detail, "db's cache has none of form 1 taken before "+fpNew)
})
if b.have("db").Fp != fpNew || b.file("db") != "v2\n" || restores.Load() != 1 {
t.Fatalf("the missing point left db %+v with %q, %d restore(s) asked", b.have("db"), b.file("db"), restores.Load())
}
// The stand-in now holds the point: the want answered "again" asks the restore, and the have waits for its ending.
pointsMu.Lock()
points = []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fpNew, Snapshot: "snap-2", At: time.Now(),
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": "again"}}}}
pointsMu.Unlock()
gated.Store(true)
calls := toolCalls.Load()
b.want("db", fpOld, "", meshwants.WantsByWantNodeModuleValueBackupAgain)
within(t, 20*time.Second, "the restore of the revert is not asked", func() bool { return restores.Load() == 2 })
within(t, 20*time.Second, "the have does not wait for the restore", func() bool { return b.have("db").Phase == nec.PhaseBackingUp })
// The restored event is held until close(release): the older build must not be applied before the stand-in says restored.
if h := b.have("db"); h.Phase != nec.PhaseBackingUp || b.file("db") != "v2\n" || toolCalls.Load() != calls {
t.Fatalf("before the stand-in says restored the have is %+v, the file %q, %d catch-up call(s) more", h, b.file("db"), toolCalls.Load()-calls)
}
close(release)
within(t, 20*time.Second, "the older build is not applied after the restore", func() bool {
h := b.have("db")
return h.Fp == fpOld && h.Phase == nec.PhaseRunning && b.file("db") == "v1\n"
})
within(t, 20*time.Second, "the catch-up tool is not called after the restore", func() bool { return toolCalls.Load() == calls+1 })
pointsMu.Lock()
asked, gotArgs := restoreAsked, toolArgs
pointsMu.Unlock()
if asked.Snapshot != "snap-2" || asked.Fingerprint != fpNew || gotArgs["all"] != "again" {
t.Errorf("the revert asked %+v and called the catch-up with %v", asked, gotArgs)
}
stopAgent()
// Design 47 G A-293: over the same session, the builder's build ends the composer follows: the stream read whole first (the
// last end of each module and fingerprint, in the stream's order), then the read's end, then each end said after it.
if _, err := b.js.AddStream(&nats.StreamConfig{Name: link.BuildEndsStream, Subjects: []string{link.BuildEndsSubject}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
ended := func(fp, outcome, why string) {
raw, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: fp, Outcome: outcome, Error: why})
if _, err := b.js.Publish("mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("db", fp), raw); err != nil {
t.Fatal(err)
}
}
ended("f1", builds.BuildFailed, "first")
ended("f1", builds.BuildFailed, "no go")
ended("f0", builds.BuildDone, "")
heard := make(chan string, 8)
follow, stopFollow := context.WithCancel(ctx)
defer stopFollow()
go func() {
_ = session.FollowBuildEnds(follow, func(e builds.BuildEnded) { heard <- e.Fingerprint + ":" + e.Outcome + ":" + e.Error },
func() { heard <- "read" })
}()
for i, want := range []string{"f1:failed:no go", "f0:done:", "read", "f2:failed:later"} {
if want == "f2:failed:later" {
ended("f2", builds.BuildFailed, "later")
}
select {
case got := <-heard:
if got != want {
t.Fatalf("build end %d heard %q, want %q", i, got, want)
}
case <-time.After(10 * time.Second):
t.Fatalf("build end %d, %q, was not heard", i, want)
}
}
}
// startSession is a session on the node's own connection, with no node agent run on it.
func (b *itBed) startSession() *link.Session {
b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
return b.session
}
cmd/mesh-host/seatlist.go
+121 -31
View File
@@ -5,11 +5,13 @@ import (
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
conditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
@@ -103,47 +105,49 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer {
return time.AfterFunc(d, f)
}
// findingsOfTheNode writes the node's findings in its state `findings`, in the layout of the seat mesh-conditions.
type findingsOfTheNode struct{ session *link.Session }
// nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they
// happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions.
type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) {
key := conditions.Key(conditions.ScopeMachine, f.ID, f.Kind)
v := condseat.FindingsByKeyValue{V: 1, Key: key, Kind: f.Kind, Observer: f.Observer, Severity: conditions.Warning,
Subject: condseat.Subject{Scope: conditions.ScopeMachine, ID: f.ID, Machine: f.ID}, Summary: f.Summary,
Headline: f.Headline, Explanation: f.Explanation, Source: "seat-list-unread", At: f.At.UTC()}
if err := conditions.CheckFinding(v); err != nil {
return "", nil, err
}
raw, err := json.Marshal(v)
return key, raw, err
// findingsOfTheNode keeps the node's findings through mesh-sdk's held writer (design 47 G §383, §384, §461, A-29): each
// finding is written in the node's state `findings` and said as an event on the engine's own stream (a foundation part's
// `mesh-host-findings`, §529, A-30), the engine says what it holds at its start and when the keeper asks, and the keeper
// reads no state of the engine's.
type findingsOfTheNode struct {
*nodeFindings
}
func (w findingsOfTheNode) Put(f views.Finding) error {
key, raw, err := w.encode(f)
// newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on
// the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own.
func newFindingsOfTheNode(session *link.Session, node, module string, say func(string)) (*findingsOfTheNode, error) {
writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: module,
Node: node,
State: link.FindingsState{Session: session},
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...),
Asked: condseat.SubjectFindingsAsked,
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
Publish: session.PublishEvent,
Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) {
return session.ListenFrom(condseat.StreamFindingsAsked, subject, handle)
},
})
if err != nil {
return err
return nil, err
}
return w.session.PutFinding(key, raw)
return &findingsOfTheNode{writer}, nil
}
// keyOf is the finding's key in the state; a read or a clear needs no more of the finding than Kind and ID.
func (w findingsOfTheNode) keyOf(f views.Finding) string {
return conditions.Key(conditions.ScopeMachine, f.ID, f.Kind)
}
func (w findingsOfTheNode) Delete(f views.Finding) error {
return w.session.DeleteFinding(w.keyOf(f))
}
// Has says whether the finding stands in the node's state `findings` now.
func (w findingsOfTheNode) Has(f views.Finding) (bool, error) {
return w.session.HasFinding(w.keyOf(f))
// Start seeds what the state holds, states the findings now and hears the keeper until ctx ends.
func (w *findingsOfTheNode) Start(ctx context.Context) error {
go func() { <-ctx.Done(); w.Close() }()
return w.nodeFindings.Start(ctx)
}
// startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what
// it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what
// the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds.
func startSeatList(ctx context.Context, session *link.Session, node string, say func(string)) (*seatListState, error) {
func startSeatList(ctx context.Context, session *link.Session, node string, findings *lateFindings, say func(string)) (*seatListState, error) {
st := &seatListState{changed: make(chan struct{}, 1)}
// Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists.
var announce atomic.Pointer[func()]
@@ -161,7 +165,7 @@ func startSeatList(ctx context.Context, session *link.Session, node string, say
return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err)
}
st.reader = reader
st.waiter = &seatwait.Waiter{Node: node, Observer: "node-engine", Reader: reader, Findings: findingsOfTheNode{session},
st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Bound: viewWaitsBound(),
Clock: seatWaitClock, Say: say}
told := func() {
st.waiter.Changed()
@@ -189,3 +193,89 @@ func startSeatList(ctx context.Context, session *link.Session, node string, say
seatListNow.Store(st)
return st, nil
}
// lateFindings is the node-engine's finding writer, made when the engine's own module is known (design 47 G §527, §634,
// A-149): at once from its applied record, or, on a first start with no such record, when the first composing names it. Till
// then a write or a clear is refused in words, never lost in silence.
type lateFindings struct {
own *link.OwnName
make func(module string) (*findingsOfTheNode, error)
mu sync.Mutex
w *findingsOfTheNode
}
// observer is the module the findings are observed as: the engine's own, "" while it is not known.
func (f *lateFindings) observer() string { return f.own.Get() }
// Start makes the writer and starts it as soon as the own name is known: in this call when it already is, else on a goroutine.
func (f *lateFindings) Start(ctx context.Context, say func(string)) {
begin := func(module string) {
w, err := f.make(module)
if err != nil { // a writer that cannot be made is a fault of its configuration: loud
say("the node-engine cannot make its finding writer: " + err.Error())
return
}
if err := w.Start(ctx); err != nil {
say("the node-engine's findings are kept in its state and not said as events until it is started again: " + err.Error())
}
f.mu.Lock()
f.w = w
f.mu.Unlock()
}
if name := f.own.Get(); name != "" {
begin(name)
return
}
go func() {
if name, err := f.own.Wait(ctx); err == nil {
begin(name)
}
}()
}
var errOwnNameUnknown = errors.New("the node-engine's own module is not yet named by its applied record or its first composing, so its finding writer is not made")
func (f *lateFindings) writer() *findingsOfTheNode {
f.mu.Lock()
defer f.mu.Unlock()
return f.w
}
func (f *lateFindings) Write(ctx context.Context, fi condseat.Finding) error {
if w := f.writer(); w != nil {
return w.Write(ctx, fi)
}
return errOwnNameUnknown
}
func (f *lateFindings) Clear(ctx context.Context, scope, id, kind, why string) error {
if w := f.writer(); w != nil {
return w.Clear(ctx, scope, id, kind, why)
}
return errOwnNameUnknown
}
func (f *lateFindings) Held() []condseat.Finding {
if w := f.writer(); w != nil {
return w.Held()
}
return nil
}
// viewWaitsFile is the file the manifest declares with the content ${config:view-waits-minutes} (design 47 G §633), read at
// the engine's start as recoveryWaitFile is.
var viewWaitsFile = "/var/lib/mesh-host/view-waits-minutes"
// viewWaitsBound is how long the reader of the seat list may wait before the engine says it: the file's minutes, the design's
// five (seatwait.Bound) when the file is missing, empty or not a positive number.
func viewWaitsBound() time.Duration {
raw, err := os.ReadFile(viewWaitsFile)
if err != nil {
return seatwait.Bound
}
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil || n <= 0 {
return seatwait.Bound
}
return time.Duration(n) * time.Minute
}
cmd/mesh-host/settled_manifest_test.go
+39
View File
@@ -26,3 +26,42 @@ func TestTheSettledEventIsTheManifestsConsume(t *testing.T) {
t.Errorf("the engine hears %s, its manifest consumes %v", settledEvent, m.Consumes)
}
}
// The manifest claims the seat node-engine, says its findings on its own stream and hears the keeper's ask and the
// node-engine seat's running (design 47 G §364, §387, §199).
func TestTheManifestClaimsItsSeatAndSaysItsFindings(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Version string `json:"version"`
Consumes []string `json:"consumes"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
} `json:"claims"`
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
} `json:"streams"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if len(m.Claims) != 1 || m.Claims[0].Name != "node-engine" || m.Claims[0].Scope != "node" {
t.Errorf("claims = %+v, want the node seat node-engine", m.Claims)
}
if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-host-findings" ||
!slices.Equal(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
t.Errorf("streams = %+v, want mesh-host-findings with the three finding events", m.Streams)
}
for _, c := range []string{"mesh-conditions.findings-asked", "node-engine.running"} {
if !slices.Contains(m.Consumes, c) {
t.Errorf("consumes %v lacks %s", m.Consumes, c)
}
}
if m.Version != "0.1" {
t.Errorf("version = %q, want MAJOR.MINOR 0.1 (§368, §411)", m.Version)
}
}
cmd/mesh-host/stop_test.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"context"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the
// record, so the module's kept data and its record survive (design 47 G A-98).
func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) {
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
path := filepath.Join(t.TempDir(), "state.json")
var s store.State
s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"})
s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"})
if err := store.Save(path, s); err != nil {
t.Fatal(err)
}
var asked []string
opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
return "", nil
}}
if res := stopComposition(context.Background(), opts, "app"); res.Failed {
t.Fatalf("stop failed: %s", res.Detail)
}
if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") ||
strings.Contains(strings.Join(asked, "\n"), " rm ") {
t.Fatalf("the machine was asked %v", asked)
}
if after, err := store.Load(path); err != nil || len(after.Resources) != 2 {
t.Fatalf("the record changed: %v, %v", after.Resources, err)
}
}
cmd/mesh-host/trust.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"github.com/novox/mesh-host/internal/buckets"
"os"
"path/filepath"
cmd/mesh-host/whatruns_test.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
engcomp "github.com/novox/mesh-host/internal/compositions"
"os"
"path/filepath"
go.mod
+13 -9
View File
@@ -8,16 +8,19 @@ require (
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
github.com/nats-io/nats-server/v2 v2.11.17
@@ -29,6 +32,7 @@ require (
require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault v0.0.0-00010101000000-000000000000
@@ -45,6 +49,6 @@ require (
)
require (
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect
)
internal/agent/agent.go
+47 -3
View File
@@ -19,7 +19,7 @@ import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/compositions"
"slices"
"sort"
@@ -75,6 +75,8 @@ type Have struct {
// PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966
// ENG-9, nodeengine.HaveByNodeModuleValue.PullWait): read through the have, as node_haves shows it.
PullWait *nodeengine.PullWait
// Wait is the hold or wait the have says, in the recovery wait with its ask-at and act (design 47 G §560).
Wait *nodeengine.Wait
}
// Haves is the node's have bucket as the agent uses it.
@@ -136,6 +138,13 @@ type Env struct {
Apply func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result
// Remove removes the module's own resources and nothing of another's.
Remove func(ctx context.Context, module string, others []compositions.Composed) Result
// Stop stops the module's units and containers and removes nothing, so what it kept survives: a revert stops the newer
// build with it, restores the module's kept data, and then applies the reverted build (design 47 G A-98, §297). Nil
// stops nothing.
Stop func(ctx context.Context, module string) Result
// Recovery takes a module's kept data as a restore point before a form change moves it and restores it at a revert
// (design 47 G §296, §297, §554); nil moves none.
Recovery *Recovery
// Backup backs a module up before a new composition of it is applied; nil backs up nothing.
Backup func(ctx context.Context, module string) Backup
// Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539
@@ -504,6 +513,9 @@ const NotComposedDetail = "the node cannot compose it, and keeps running what it
// composerMark says h is a mark NotComposed wrote: failed, its detail opening with NotComposedDetail.
func composerMark(h Have) bool {
if h.Phase == string(nodeengineclient.PhaseHeld) && heldForEntry(h.Detail) {
return true
}
return h.Phase == string(nodeengineclient.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail)
}
@@ -647,8 +659,14 @@ func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string
a.mu.Lock()
a.notComposed[module] = mark
a.mu.Unlock()
phase, detail := string(nodeengineclient.PhaseFailed), NotComposedDetail+why
if heldForEntry(why) {
// The entry its want names is not held: the phase held, its detail opening entry-asked: or entry-missing: and
// naming the fingerprint (design 47 G section 421).
phase, detail = string(nodeengineclient.PhaseHeld), why
}
a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
Phase: string(nodeengineclient.PhaseFailed), Detail: NotComposedDetail + why})
Phase: phase, Detail: detail})
}
}
@@ -1423,7 +1441,8 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
// be swallowed by "it runs what its want names" (issue 1585, the second failure). The operator's answer, again or
// skip, in a revision after the one the hold took, is acted on whatever runs; any other new revision of the want
// is the answer only for a move (ADR 0568 §4), as before.
answered := w.Rev > att.AnsweredRev && (w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupAgain || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip)
answered := w.Rev > att.AnsweredRev && (w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupAgain || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip ||
w.Want.Revert == meshwants.WantsByWantNodeModuleValueRevertLive)
if !answered && (runsWant || (att.AnsweredRev >= w.Rev && att.CompRev == comp.Rev)) {
a.sayOnce("held:"+module, fmt.Sprintf("%s is held for a failed backup before %s: it waits for the operator's "+
"answer, and its want (revision %d) carries none since the hold (revision %d)", module, w.Want.Fp, w.Rev, att.AnsweredRev))
@@ -1601,6 +1620,30 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
ranBefore = ranBefore || a.prior[module]
a.mu.Unlock()
// A want with parts is applied only when they and the digests its composition fetches agree (design 47 G §524): the
// module keeps running what it has, and the have says which digest differs.
if words := partsDifference(w.Want.Parts, comp.C.Resources); words != "" {
a.mu.Lock()
prev, runs := a.running[module]
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
a.mu.Unlock()
if !runs {
if h, found, _ := a.readHave(ctx, module); found {
prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed}
}
}
a.env.Say(fmt.Sprintf("%s is held: %s", module, words))
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld),
Detail: words})
return
}
// A kept data on the bus whose form changes is taken as a restore point first, or restored at a revert (design 47 G §296, §297).
held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore, others)
if held {
return
}
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
// without one (ADR 0539 §16).
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != meshwants.WantsByWantNodeModuleValueBackupSkip {
@@ -1705,6 +1748,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
a.env.Settled(module)
}
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseRunning), Detail: res.Detail})
a.catchUp(ctx, module, catchUps)
}
// ranFailed marks what runs of a module as failed once an apply of it failed (novox/hq issue 1966, 2026-11-08): a failed
internal/agent/agent_test.go
+53 -2
View File
@@ -6,8 +6,8 @@ import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"github.com/novox/mesh-host/internal/compositions"
"github.com/novox/mesh-host/internal/judge"
@@ -322,6 +322,50 @@ func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) {
if got := b.appliedList(); len(got) != 1 {
t.Fatalf("a want naming another digest applied the composition: %v", got)
}
// Design 47 G section 524: a want with parts is applied only when its parts and the digests the composition fetches
// agree; a want with no parts is applied as before.
d1, d2, d3 := "sha256:"+strings.Repeat("1", 64), "sha256:"+strings.Repeat("2", 64), "sha256:"+strings.Repeat("3", 64)
archive := func(id, digest string) json.RawMessage {
raw, _ := json.Marshal(map[string]any{"id": "web." + id, "type": "archive", "path": "/tmp/r6/web/" + id,
"source": "https://store/" + id, "digest": digest, "module": "web"})
return raw
}
for i, row := range []struct {
name string
parts map[string]string
applies bool
detail string
}{
{"agree", map[string]string{"web:a": d1, "web:b": d2}, true, ""},
{"no parts", nil, true, ""},
{"fetched outside", map[string]string{"web:a": d1}, false, PartsDiffer + " fetched and not in the want's parts: " + d2},
{"part not fetched", map[string]string{"web:a": d1, "web:b": d2, "web:c": d3}, false,
PartsDiffer + " in the want's parts and not fetched: " + d3},
} {
before := len(b.appliedList())
fpn := fpOf(string("5678"[i]))
ckn, crawn, dign := composition(t, "web", fpn, false, archive("a", d1), archive("b", d2))
b.comps.put(ckn, crawn)
wkn := meshwants.WantsByWantNodeModuleKey(node, "web")
wrawn, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpn, Composed: dign, Parts: row.parts})
if err != nil {
t.Fatal(err)
}
b.wants.put(wkn, wrawn)
b.idle()
if got := len(b.appliedList()) - before; (got == 1) != row.applies {
t.Fatalf("parts %q: applied %d times, want applies=%v", row.name, got, row.applies)
}
if h := b.haves.last("web"); !row.applies && (h.Phase != string(nodeengineclient.PhaseHeld) || h.Detail != row.detail) {
t.Fatalf("parts %q: the have is %s %q, want held %q", row.name, h.Phase, h.Detail, row.detail)
}
}
// A container's digest is the one after its image's @.
container := json.RawMessage(`{"id":"web.c","type":"container","name":"web-c","image":"store/web@` + d3 + `","module":"web"}`)
if got := partsDifference(map[string]string{"web:c": d3}, []json.RawMessage{container}); got != "" {
t.Fatalf("a container whose digest is in the parts differs: %q", got)
}
}
// R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied
@@ -886,6 +930,7 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange)
b := newBed(t)
stop := b.run()
defer stop()
@@ -1728,6 +1773,12 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
if h := b.haves.last("web"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != dig {
t.Errorf("web, which composes, is marked beside idle: %+v", h)
}
// Design 47 G §421: a module whose want's entry the catalog half holds none of is the phase held, its detail opening
// entry-missing: and naming the fingerprint; composed again, its have is what it runs.
b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": EntryMissing + " the catalog half holds no entry of the build src2:x"})
if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseHeld) || h.Composed != idleDig || h.Detail != EntryMissing+" the catalog half holds no entry of the build src2:x" {
t.Errorf("idle's have while its entry is missing: %+v", h)
}
b.ag.NotComposedEach(context.Background(), "", nil)
if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != idleDig {
t.Errorf("idle's have once it composes again: %+v", h)
internal/agent/composition.go
+11
View File
@@ -44,6 +44,17 @@ type CompositionView struct {
// Own is, of the node's own composition (module node), its own fields beside what runs names (issue 774: ace's
// own composition dropped the provisioner client's account, and nothing read could show it).
Own *OwnView `json:"own,omitempty"`
// Withheld are the module's marked subjects the grants part withholds from this node's runtime, each with why, read
// through the seat mesh-grants' verb withheld; WithheldUnknown says when that verb did not answer (design 46 approval
// mark §6, lead answer A-239).
Withheld []WithheldView `json:"withheld,omitempty"`
WithheldUnknown string `json:"withheld-unknown,omitempty"`
}
// WithheldView is one marked subject withheld, and why.
type WithheldView struct {
Subject string `json:"subject"`
Why string `json:"why"`
}
// OwnView is the node's own fields of a composition (engcomp.NodeOwn) beside what runs of them, and what the node's
internal/agent/kept.go
+372
View File
@@ -0,0 +1,372 @@
package agent
import (
"context"
"fmt"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-host/internal/compositions"
)
// A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G
// §296, §297, §554, §555, §560, §630 to §632, the lead's answers A-77, A-98 and A-108): the node agent asks the seat
// `mesh-recovery`'s `snapshot-kept` and waits for the `ended` of its own act, or, at a revert, stops the newer build, asks
// `kept-points`, asks `restore-kept` with the snapshot of the point of the form it reverts from, waits for its `restored`, and
// after the reverted build's apply calls the point's catch-up as the module's tool.
// Ending is the ending event of an act of the recovery module as the node agent reads it: `ended` for a restore point taken,
// `restored` for one restored (design 47 G §560).
type Ending struct {
Act string
Outcome string
Error string
Snapshot string
// Why is the hand act's words for a cancel (design 47 G A-117, §560).
Why string
}
// RecoveryWaitMinutes is the default of the manifest's config key `recovery-wait-minutes`, the bound of the wait for an
// ending event, counted from the first ask (design 47 G §554, §633).
const RecoveryWaitMinutes = 90
// Recovery is what the node agent needs of the recovery module and the manifests to move kept data. Nil moves none.
type Recovery struct {
// FormChanges are the kept data whose form differs between the running build's manifest and the applied build's (the
// catalog half's entries by fingerprint); an error says it is not known.
FormChanges func(ctx context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error)
// The seat's verbs, asked in the node-engine's own name; an error is the ask not answered.
SnapshotKept func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error)
RestoreKept func(ctx context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error)
KeptPoints func(ctx context.Context, in meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error)
// Hear follows the ending events of kind "ended" or "restored" of one module's kept data: the last one said already first,
// then each after it, until stop is called. It is made before the verb is asked (design 47 G §560).
Hear func(ctx context.Context, kind, module, kept string) (events <-chan Ending, stop func(), err error)
// Bound writes the bound key `<what>.<id>` of a wait in the state bounds with its time to live, so its end is the bus's
// marker, heard as expired (design 47 G A-77, A-92); end deletes it. Nil keeps no bound.
Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error)
// CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108).
CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error
// WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes.
WaitMinutes int
// Now is the clock; nil reads the machine's.
Now func() time.Time
}
func (r *Recovery) now() time.Time {
if r.Now != nil {
return r.Now()
}
return time.Now()
}
func (r *Recovery) bound() time.Duration {
if r.WaitMinutes > 0 {
return time.Duration(r.WaitMinutes) * time.Minute
}
return RecoveryWaitMinutes * time.Minute
}
// KeptMove is one kept data a want moves across a change of its form, as the seat mesh-recovery's verbs name it.
type KeptMove struct {
// Name is the kept data's local name, Form the form the restore point holds (the older one), and Fingerprint the
// fingerprint the restore point is taken before (the newer build's) or reverted from.
Name string
Form int
Fingerprint string
// CatchUp is, when taking the point, the tool the newer manifest declares beside the kept data.
CatchUp *meshrecovery.CatchUp
}
// keptMoves are the restore points a move needs: a point to take before the newer build starts (the kept data's form rises,
// or the newer manifest declares it no more) and a point to restore before the older build starts (the kept data goes back to
// a lower form). A kept data the running build does not declare is neither: nothing was written before it.
func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (take, restore []KeptMove) {
for _, c := range changes {
switch {
case c.Running == nil:
case c.Applied == nil || c.Applied.Form > c.Running.Form:
m := KeptMove{Name: c.Name, Form: c.Running.Form, Fingerprint: appliedFp}
if c.Applied != nil && c.Applied.CatchUp != nil {
m.CatchUp = &meshrecovery.CatchUp{Tool: c.Applied.CatchUp.Tool, Arguments: c.Applied.CatchUp.Arguments}
}
take = append(take, m)
case c.Applied.Form < c.Running.Form:
restore = append(restore, KeptMove{Name: c.Name, Form: c.Applied.Form, Fingerprint: runningFp})
}
}
return take, restore
}
// recoveryHeld is how a wait for the recovery module ended when the move cannot go on: the words are the reason of the hold.
type recoveryHeld struct {
Words string
Interrupted bool
}
// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
// ending of the act it names and asks again, `failed` with `stopped` asks again within the same bound, a failure with any
// other error holds. The wait is written in the have with its first ask's time and the act, so a restart waits for that act
// and asks nothing again; past the bound the have says the wait, and the wait goes on (the watchman raises it).
func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, ask func() (accepted, busy, errWords string, err error)) (Ending, *recoveryHeld) {
r := a.env.Recovery
module := h.Module
step := nodeengine.WaitStepCompose
wait := nodeengine.Wait{Owner: nodeengine.Seat, Step: step, On: "mesh-recovery." + kind + "." + module + "." + k.Name,
Reason: "recovery waits", Bound: r.bound().String()}
// A restart: the have names the wait with its ask time and act.
last, ok := a.env.Haves.Read(ctx, module)
if ok && last.Wait != nil && last.Wait.On == wait.On && last.Wait.AskAt != nil {
wait.AskAt, wait.Act, wait.Since = last.Wait.AskAt, last.Wait.Act, last.Wait.Since
}
now := r.now()
if wait.AskAt == nil {
wait.AskAt, wait.Since = &now, now
}
events, stop, err := r.Hear(ctx, kind, module, k.Name)
if err != nil {
return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act cannot be heard: " + err.Error()}
}
defer stop()
var expired <-chan struct{}
if r.Bound != nil {
bound := r.bound() - now.Sub(*wait.AskAt)
if bound < time.Second {
bound = time.Second
}
exp, end, berr := r.Bound(ctx, "recovery", module+"."+kind+"."+k.Name, module, *wait.AskAt, bound)
if berr != nil {
return Ending{}, &recoveryHeld{Words: "the bound of the wait for the recovery module cannot be written: " + berr.Error()}
}
defer end()
expired = exp
}
write := func() {
w := wait
h.Wait = &w
a.writeHave(ctx, h)
}
act, busyOn := wait.Act, ""
for {
if act == "" && busyOn == "" {
write() // the first ask's time is in the have before the ask
accepted, busy, words, err := ask()
switch {
case err != nil:
if ctx.Err() != nil {
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
}
return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()}
case words != "":
return Ending{}, &recoveryHeld{Words: words}
case accepted != "":
act, wait.Act = accepted, accepted
write()
case busy != "":
busyOn = busy
default:
return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"}
}
}
select {
case <-ctx.Done():
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
case <-expired:
expired = nil
wait.Words = fmt.Sprintf("the recovery module has not ended its act within %s", r.bound())
write()
case ev, open := <-events:
if !open {
return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act is no longer heard"}
}
switch {
case busyOn != "" && ev.Act == busyOn:
busyOn = ""
case act != "" && ev.Act == act:
switch {
case ev.Outcome == meshrecovery.EndedEventOutcomeDone:
return ev, nil
case ev.Error == "stopped":
act, wait.Act = "", ""
case ev.Error == "cancelled":
words := "cancelled"
if ev.Why != "" {
words += ": " + ev.Why
}
return ev, &recoveryHeld{Words: words}
default:
words := ev.Error
if words == "" {
words = "the recovery module's act failed"
}
return ev, &recoveryHeld{Words: words}
}
}
}
}
}
// keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it
// answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up
// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither, and nor does one that answers a held revert live.
func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool, others []compositions.Composed) (held bool, catchUps []meshrecovery.CatchUp) {
r := a.env.Recovery
if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip {
return false, nil
}
fp, dig := w.Want.Fp, w.Want.Composed
a.mu.Lock()
prev, runs := a.running[module]
a.mu.Unlock()
if !runs {
if h, found, _ := a.readHave(ctx, module); found {
prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed}
}
}
if prev.Fp == "" || prev.Fp == fp {
return false, nil
}
changes, err := r.FormChanges(ctx, module, prev.Fp, fp)
if err != nil {
// Whether the move changes a form is not known: it waits for the entries and never moves without its restore point (design 47 G
// A-113). No attempt is kept, so the composer handing the composition again, or any settle, asks again.
words := fmt.Sprintf("whether %s's kept data changes form between %s and %s is not known, so it is not moved: %v", module, prev.Fp, fp, err)
a.sayOnce("kept:"+module+":"+fp, module+" waits: "+words)
since := r.now()
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: since, Words: words}})
if a.env.Waiting != nil {
a.env.Waiting(module, fp)
}
return true, nil
}
take, restore := keptMoves(changes, prev.Fp, fp)
if len(take)+len(restore) == 0 {
return false, nil
}
// The operator's answer "live" to a held revert, read from this want and from nothing else: the reverted build starts on the live
// data as it is, with no restore, no aside taken and no catch-up, and applies as any want (design 47 G section 710).
if len(restore) > 0 && w.Want.Revert == meshwants.WantsByWantNodeModuleValueRevertLive {
a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert))
return false, nil
}
hold := func(words string, interrupted bool) {
if interrupted || ctx.Err() != nil {
words = BackupInterrupted
a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
} else {
a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
}
a.mu.Lock()
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
a.mu.Unlock()
since := r.now()
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}})
}
backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)}
for _, m := range take {
why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp)
_, held := a.askAndWait(ctx, backing, "ended", m, func() (string, string, string, error) {
got, err := r.SnapshotKept(ctx, meshrecovery.SnapshotKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
Fingerprint: m.Fingerprint, CatchUp: m.CatchUp, Why: why})
return got.Accepted, got.Busy, got.Error, err
})
if held != nil {
hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
return true, nil
}
}
if len(restore) == 0 {
return false, nil
}
// A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside.
stopped := false
if a.env.Stop != nil {
if res := a.env.Stop(ctx, module); res.Failed {
hold("the newer build could not be stopped before the restore: "+res.Detail, false)
return true, nil
}
stopped = true
}
// A revert that cannot restore starts the newer build again and holds the move, saying why: a module is never left stopped
// (design 47 G A-113).
holdRevert := func(words string, interrupted bool) {
if stopped {
words += "; " + a.startAgain(ctx, module, prev, others)
}
hold(words, interrupted)
}
for _, m := range restore {
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name})
if err != nil {
holdRevert(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false)
return true, nil
}
var point *meshrecovery.Point
for i := range pts.Points {
p := pts.Points[i]
if p.Module == module && p.Kept == m.Name && int(p.Form) == m.Form && p.Fingerprint == m.Fingerprint {
if point == nil || p.At.After(point.At) {
point = &pts.Points[i]
}
}
}
if point == nil {
holdRevert(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false)
return true, nil
}
why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint)
_, held := a.askAndWait(ctx, backing, "restored", m, func() (string, string, string, error) {
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
Fingerprint: m.Fingerprint, Snapshot: point.Snapshot, Why: why})
return got.Accepted, got.Busy, got.Error, err
})
if held != nil {
holdRevert(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
return true, nil
}
if point.CatchUp != nil {
catchUps = append(catchUps, *point.CatchUp)
}
}
return false, catchUps
}
// startAgain starts the build a module ran before a revert stopped it, from the composition the agent applied or holds of its
// fingerprint, and says whether it did (design 47 G A-113).
func (a *Agent) startAgain(ctx context.Context, module string, prev nodeengine.HaveByNodeModuleValue, others []compositions.Composed) string {
a.mu.Lock()
c, ok := a.applied[module]
if !ok || c.C.Module == "" {
c, ok = a.compositionFor(module, prev.Fp)
}
a.mu.Unlock()
if !ok || a.env.Apply == nil {
return "the newer build could not be started again: its composition is not held"
}
res := a.env.Apply(context.WithoutCancel(ctx), c.C, others)
if res.Failed {
return "the newer build could not be started again: " + res.Detail
}
return "the newer build runs again"
}
// catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own
// tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand.
func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) {
if a.env.Recovery == nil || a.env.Recovery.CatchUp == nil {
return
}
for _, c := range calls {
if err := a.env.Recovery.CatchUp(ctx, module, c); err != nil {
a.env.Say(fmt.Sprintf("%s runs the reverted build, and its catch-up tool %s failed: %v", module, c.Tool, err))
continue
}
a.env.Say(fmt.Sprintf("%s's catch-up tool %s ran after the restore", module, c.Tool))
}
}
internal/agent/kept_test.go
+319
View File
@@ -0,0 +1,319 @@
package agent
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"github.com/novox/mesh-host/internal/compositions"
)
// A recovery module that answers as scripted: each ask takes the next answer and says the events it carries after it.
type recoveryFake struct {
mu sync.Mutex
asks []string // verb and its input, in order
answers []fakeAnswer
points []meshrecovery.Point
events chan Ending
changes []catalogue.FormChange
calls []string // stop, ask, apply, catch-up, in order
catchUp []meshrecovery.CatchUp
expired chan struct{}
unknown error // FormChanges answers it
}
type fakeAnswer struct {
accepted, busy, errWords string
then []Ending
}
func (f *recoveryFake) next(verb string, in any) (string, string, string) {
f.mu.Lock()
defer f.mu.Unlock()
f.asks = append(f.asks, fmt.Sprintf("%s %s", verb, asJSON(in)))
f.calls = append(f.calls, verb)
if len(f.answers) == 0 {
return "", "", "no answer scripted"
}
a := f.answers[0]
f.answers = f.answers[1:]
for _, e := range a.then {
f.events <- e
}
return a.accepted, a.busy, a.errWords
}
func (f *recoveryFake) env() *Recovery {
return &Recovery{
FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) {
return f.changes, f.unknown
},
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
a, b, e := f.next("snapshot-kept", in)
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
},
RestoreKept: func(_ context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) {
a, b, e := f.next("restore-kept", in)
return meshrecovery.RestoreKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
},
KeptPoints: func(context.Context, meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, "kept-points")
return meshrecovery.KeptPointsAnswer{Points: f.points}, nil
},
Hear: func(context.Context, string, string, string) (<-chan Ending, func(), error) {
return f.events, func() {}, nil
},
Bound: func(context.Context, string, string, string, time.Time, time.Duration) (<-chan struct{}, func(), error) {
return f.expired, func() {}, nil
},
CatchUp: func(_ context.Context, _ string, c meshrecovery.CatchUp) error {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, "catch-up")
f.catchUp = append(f.catchUp, c)
return nil
},
Now: time.Now,
}
}
func (f *recoveryFake) seen() []string {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.calls)
}
// moveDB runs a data module at fp "a", then wants it at fp "b", and answers the bed once the second want is settled or held.
func moveDB(t *testing.T, f *recoveryFake, ranFrom, to string, settle func(b *bed) bool, before ...func(b *bed, dig string)) *bed {
t.Helper()
b := newBed(t)
inner, applies := b.env.Apply, 0
b.env.Apply = func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result {
if applies++; applies >= 2 {
f.mu.Lock()
f.calls = append(f.calls, "apply")
f.mu.Unlock()
}
return inner(ctx, c, others)
}
f.events, f.expired = make(chan Ending, 16), make(chan struct{})
b.env.Recovery = f.env()
b.env.Stop = func(context.Context, string) Result {
f.mu.Lock()
f.calls = append(f.calls, "stop")
f.mu.Unlock()
return Result{}
}
stop := b.run()
t.Cleanup(stop)
fp1, fp2 := fpOf(ranFrom), fpOf(to)
ck, craw, dig := composition(t, "db", fp1, true, file("db", "x", "1"))
wk, wraw := wantFor(t, "db", fp1, dig, "")
b.comps.put(ck, craw)
b.wants.put(wk, wraw)
eventually(t, "the first build is not applied", func() bool { return len(b.appliedList()) == 1 })
b.idle()
ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
for _, h := range before {
h(b, dig)
}
wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
b.comps.put(ck2, craw2)
b.wants.put(wk2, wraw2)
eventually(t, "the move is neither applied nor held", func() bool { return settle(b) })
return b
}
// testAMoveAcrossAFormChange is design 47 G §296, §297, §554, §560, §630 to §632 and A-98, A-108 at the node agent.
func testAMoveAcrossAFormChange(t *testing.T) {
up := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
Applied: &catalogue.KeptLayout{Form: 2, CatchUp: &catalogue.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
down := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 2},
Applied: &catalogue.KeptLayout{Form: 1}}}
fp := func(c string) string { return fpOf(c) }
done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} }
failed := func(act, why string) Ending {
e := Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why}
if why == "cancelled" {
e.Why = "the operator stopped it" // the hand act's words (design 47 G A-117)
}
return e
}
applied := func(b *bed) bool { return len(b.appliedList()) == 2 }
heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" }
t.Run("a form that rises is taken as a restore point before the move, with the why and the catch-up, and then applied", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{done("a1")}}}}
moveDB(t, f, "a", "b", applied)
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
want := fmt.Sprintf("snapshot-kept %s", asJSON(meshrecovery.SnapshotKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("b"),
Why: "form change of db's cache to " + fp("b"), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}))
if f.asks[0] != want {
t.Fatalf("asked %s, want %s", f.asks[0], want)
}
})
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
{accepted: "a2", then: []Ending{failed("a2", "stopped")}},
{accepted: "a3", then: []Ending{done("a3")}},
}}
moveDB(t, f, "a", "b", applied)
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "snapshot-kept", "snapshot-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
})
for _, row := range []struct{ error, held string }{
{"cancelled", "cancelled: the operator stopped it"},
{"cap", "cap"},
{"restore point damaged", "restore point damaged"},
{"disk full", "disk full"},
} {
t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" {
t.Fatalf("the have %+v", h)
}
if got := f.seen(); len(got) != 1 {
t.Fatalf("asked again: %v", got)
}
if n := len(b.appliedList()); n != 1 {
t.Fatalf("applied %d", n)
}
})
}
t.Run("the bound passed says the wait in the have and the wait goes on to the ending", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1"}}}
b := moveDB(t, f, "a", "b", func(b *bed) bool {
b.haves.mu.Lock()
defer b.haves.mu.Unlock()
for _, h := range b.haves.log {
if h.Wait != nil && h.Wait.Words != "" && h.Phase == "backing-up" {
return true
}
}
return false
}, func(*bed, string) { close(f.expired) })
f.events <- done("a1")
eventually(t, "the move went on after the ending", func() bool { return applied(b) })
})
t.Run("a revert stops the newer build, restores the point of the form it reverts from, applies, then calls its catch-up", func(t *testing.T) {
f := &recoveryFake{changes: down,
points: []meshrecovery.Point{
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("z"), Snapshot: "other", At: time.Now()},
{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now(), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild"}}},
answers: []fakeAnswer{{accepted: "r1", then: []Ending{done("r1")}}}}
moveDB(t, f, "a", "b", func(b *bed) bool { return len(f.seen()) >= 5 })
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply", "catch-up"}) {
t.Fatalf("calls %v", got)
}
if f.catchUp[0].Tool != "rebuild" {
t.Fatalf("catch-up %+v", f.catchUp)
}
want := fmt.Sprintf("restore-kept %s", asJSON(meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"),
Snapshot: "snap-1", Why: "revert of db's cache from " + fp("a")}))
if f.asks[0] != want {
t.Fatalf("asked %s, want %s", f.asks[0], want)
}
})
t.Run("a revert with no point of its form holds as a restore point missing", func(t *testing.T) {
f := &recoveryFake{changes: down}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
t.Fatalf("the have %+v", h)
}
})
t.Run("a held revert, then the want with revert live, starts the older build with no restore, no aside and no catch-up (design 47 G section 710)", func(t *testing.T) {
f := &recoveryFake{changes: down}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
t.Fatalf("the have %+v", h)
}
before := len(b.appliedList())
_, _, dig2 := composition(t, "db", fpOf("b"), true, file("db", "x", "2"))
raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpOf("b"), Composed: dig2, Revert: meshwants.WantsByWantNodeModuleValueRevertLive,
Words: "the operator answered \"live\" to the revert ask (asking 1)"})
if err != nil {
t.Fatal(err)
}
b.wants.put(meshwants.WantsByWantNodeModuleKey(node, "db"), raw)
eventually(t, "the older build runs on the answer", func() bool { h := b.haves.last("db"); return h.FP == fpOf("b") && h.Phase != "held" })
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "apply", "apply"}) && len(b.appliedList()) <= before {
t.Fatalf("calls %v", got)
}
for _, c := range f.seen() {
if c == "restore-kept" || c == "snapshot-kept" || c == "catch-up" {
t.Fatalf("calls %v", f.seen())
}
}
if n := strings.Count(strings.Join(f.seen(), " "), "kept-points"); n != 1 {
t.Fatalf("the points were read again on the answer: %v", f.seen())
}
})
t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) {
f := &recoveryFake{changes: down,
points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}},
answers: []fakeAnswer{{accepted: "r1", then: []Ending{failed("r1", "restore point damaged")}}}}
b := moveDB(t, f, "a", "b", heldHave)
if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
if h := b.haves.last("db"); !contains(h.Detail, "restore point damaged") || !contains(h.Detail, "the newer build runs again") ||
h.Wait == nil || h.Wait.Step != nodeengine.WaitStepCompose {
t.Fatalf("the have %+v", h)
}
})
t.Run("a form change not known moves nothing and waits for the entries (A-113)", func(t *testing.T) {
f := &recoveryFake{unknown: fmt.Errorf("the entry is asked")}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, "is not known") || h.Wait == nil || h.Wait.Reason != "entry-missing" {
t.Fatalf("the have %+v", h)
}
if n := len(b.appliedList()); n != 1 || len(f.seen()) != 0 {
t.Fatalf("applied %d, calls %v", n, f.seen())
}
})
t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) {
f := &recoveryFake{changes: up}
b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) {
at := time.Now().Add(-time.Minute)
b.haves.mu.Lock()
b.haves.stored["db"] = Have{Module: "db", FP: fp("a"), Phase: "backing-up", Wait: &nodeengine.Wait{
On: "mesh-recovery.ended.db.cache", AskAt: &at, Act: "a1", Since: at}}
b.haves.mu.Unlock()
f.events <- done("a1")
})
_ = b
if got := f.seen(); !slices.Equal(got, []string{"apply"}) {
t.Fatalf("calls %v", got)
}
})
}
func contains(s, sub string) bool { return strings.Contains(s, sub) }
func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) }
internal/agent/parts.go
+89
View File
@@ -0,0 +1,89 @@
package agent
import (
"encoding/json"
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
// PartsDiffer is the start of the detail of a have held because a want's parts and the digests its composition fetches
// disagree (design 47 G section 524).
const PartsDiffer = "parts-differ:"
// EntryAsked and EntryMissing open the detail of a have held for the entry the want names (design 47 G section 421 and
// section 614): the entry is asked of the catalog half and not yet answered, or the half answered that it holds none. Both
// are the phase held, and each names the fingerprint.
const (
EntryAsked = "entry-asked:"
EntryMissing = "entry-missing:"
)
// heldForEntry says whether why is a hold for the entry a want names.
func heldForEntry(why string) bool {
return strings.HasPrefix(why, EntryAsked) || strings.HasPrefix(why, EntryMissing)
}
// partsDifference checks a want's `parts` against the digests of the archive, process and container resources of the
// composition, which are the digests the apply fetches and checks (design 47 G section 524). A want with no `parts` (an
// assigner of before G, or a node whose record does not state `want-parts`) is applied as today: it returns "". With
// `parts`, the want is applied only when the digest of every resource fetched is among the values of `parts` and every
// value of `parts` is the digest of a resource fetched; otherwise it returns the detail naming each digest.
func partsDifference(parts map[string]string, resources []json.RawMessage) string {
if len(parts) == 0 {
return ""
}
fetched := map[string]bool{}
for _, raw := range resources {
r, err := declaration.ParseResource(raw)
if err != nil {
continue
}
switch res := r.(type) {
case *declaration.Archive:
if res.Source != "" {
fetched[res.Digest] = true
}
case *declaration.Process:
if res.Source != "" {
fetched[res.Digest] = true
}
case *declaration.Container:
if strings.HasPrefix(res.Image, "sha256:") {
fetched[res.Image] = true
} else if _, digest, ok := strings.Cut(res.Image, "@"); ok {
fetched[digest] = true
}
}
}
named := map[string]bool{}
for _, d := range parts {
named[d] = true
}
var outside, missing []string
for d := range fetched {
if !named[d] {
outside = append(outside, d)
}
}
for d := range named {
if !fetched[d] {
missing = append(missing, d)
}
}
if len(outside) == 0 && len(missing) == 0 {
return ""
}
sort.Strings(outside)
sort.Strings(missing)
var said []string
if len(outside) > 0 {
said = append(said, "fetched and not in the want's parts: "+strings.Join(outside, ", "))
}
if len(missing) > 0 {
said = append(said, "in the want's parts and not fetched: "+strings.Join(missing, ", "))
}
return fmt.Sprintf("%s %s", PartsDiffer, strings.Join(said, "; "))
}
internal/apply/apply.go
+5 -2
View File
@@ -785,7 +785,7 @@ func applyScoped(
// change one apply late, and never misses it.
// And one patience with the artifact store for the whole apply (novox/hq issue 291): a fetch it
// does not answer during a maintenance window waits for the window instead of failing.
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
groups: wanted, handed: handed}
// Where nothing is placed or mounted from, judged before anything is applied (novox/hq issue 339), and
// which declared directories were already here: one this apply makes is the mesh's, one found may not be.
@@ -1625,7 +1625,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
if err != nil {
return begin(r), err
}
return applyProcess(ctx, res, run, changed, previous, held, in.away)
return applyProcess(ctx, withNode(res, in.node), run, changed, previous, held, in.away)
case *declaration.Action:
return applyAction(ctx, res, run)
case *declaration.Network:
@@ -2816,6 +2816,9 @@ type inputs struct {
// fetched is every archive's and process's bytes this apply fetched before anything was replaced (fetchFirst), each
// taken by its applier once (artifact).
fetched map[string]fetchedArtifact
// node is the node's own name, the declaration's For: every process runs with it as MESH_NODE beside its own env
// (design 47 G A-342). Empty where the declaration names none (the first node's carried bundle).
node string
}
// fileDigest is what a file the container reads holds, by digest.
internal/apply/fetch_first.go
+14 -3
View File
@@ -60,10 +60,21 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat
continue
}
}
got, seen := bySource[source]
if !seen {
// What the module's done build kept on a build node is fetched from there, not from the store (design 47).
cacheKey, kept := source, false
if kf := keptFetchFrom(ctx); kf != nil && ours {
cacheKey = "kept " + module + " " + digest
if _, seen := bySource[cacheKey]; seen {
kept = true
} else if body, isKept, err := kf(ctx, module, digest); isKept {
kept = true
bySource[cacheKey] = fetchedArtifact{body: body, err: err}
}
}
got, seen := bySource[cacheKey]
if !seen && !kept {
got.body, got.err = fetchMinding(ctx, source, in.away)
bySource[source] = got
bySource[cacheKey] = got
}
if got.err == nil {
sum := sha256.Sum256(got.body)
internal/apply/filled.go
+1 -1
View File
@@ -95,7 +95,7 @@ func WriteFilledWith(
"yet taken (%s), so nothing was written", path, held.Why)}
}
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
groups: groupsWanted(d.Resources), handed: Handed{}}
in.refused = refusedPlaces(d.Resources)
in.agentHomes = rootNeverHomes(d)
internal/apply/hold_test.go
+85 -37
View File
@@ -283,7 +283,7 @@ func predecessor(t *testing.T) (dir, page string, m *machine) {
}}
}
func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
func caseAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
dir, page, m := predecessor(t)
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
@@ -318,7 +318,7 @@ func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) {
}
}
func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
func caseAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
dir, page, m := predecessor(t)
report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
@@ -338,7 +338,7 @@ func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) {
}
}
func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
func caseWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
// Assigning prepares: what the module declares that is not there is made.
dir := t.TempDir()
page := filepath.Join(dir, "index.html")
@@ -355,7 +355,7 @@ func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) {
}
}
func TestAFileThisHostWroteIsNotFound(t *testing.T) {
func caseAFileThisHostWroteIsNotFound(t *testing.T) {
// Found means present with no record. A record of any origin — carried or declared, this life
// of the node or an earlier one — means this host wrote it.
for _, origin := range []string{store.OriginCarried, store.OriginDeclared} {
@@ -372,7 +372,7 @@ func TestAFileThisHostWroteIsNotFound(t *testing.T) {
}
}
func TestAContainerAHostMadeIsNotFound(t *testing.T) {
func caseAContainerAHostMadeIsNotFound(t *testing.T) {
dir, page, m := predecessor(t)
m.containers["hello-web"].spec = "some-spec"
report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
@@ -381,7 +381,7 @@ func TestAContainerAHostMadeIsNotFound(t *testing.T) {
}
}
func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
func caseTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
// ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted
// as a module by name. It is the mesh's own and must never read as a predecessor's.
dir := t.TempDir()
@@ -398,7 +398,7 @@ func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) {
}
}
func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
func caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
h, _ := state.HeldAt("hello-web.page")
@@ -426,7 +426,7 @@ func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) {
}
}
func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
func caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
@@ -447,7 +447,7 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) {
}
}
func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
func caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
first, _ := state.HeldAt("hello-web.page")
@@ -482,7 +482,7 @@ func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.
}
}
func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
func caseACarriedApplyLetsNoHoldGo(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir)
carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
@@ -496,7 +496,7 @@ func TestACarriedApplyLetsNoHoldGo(t *testing.T) {
}
}
func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
func caseAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
@@ -520,7 +520,7 @@ func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) {
}
}
func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
func caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) {
for _, c := range []struct {
change func(*machine)
want string
@@ -547,7 +547,7 @@ func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T)
}
}
func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
func caseAHeldFileThatVanishesIsNotCreated(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untakenWeb, webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
@@ -563,7 +563,7 @@ func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) {
}
}
func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
func caseAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
// No adoption, no holds: byte for byte what a converged node did before ADR 0100.
dir, page, m := predecessor(t)
d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`)
@@ -585,7 +585,7 @@ func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) {
}
}
func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
func caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "nftables.conf")
_ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644)
@@ -621,7 +621,7 @@ func untaken(ids ...string) string {
return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}`
}
func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
func caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "data")
if err := os.Mkdir(data, 0o700); err != nil {
@@ -652,7 +652,7 @@ func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing
}
}
func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
func caseADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
// A file's parent is made as the file is written; what the mesh made is not found.
dir := t.TempDir()
data := filepath.Join(dir, "data")
@@ -671,7 +671,7 @@ func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) {
}
}
func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
func caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "hello.conf")
m := &machine{containers: map[string]*fakeContainer{},
@@ -695,7 +695,7 @@ func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t
}
}
func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
func caseAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
// A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service.
dir := t.TempDir()
conf := filepath.Join(dir, "daemon.json")
@@ -716,7 +716,7 @@ func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) {
}
}
func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
func caseAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
// The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit
// the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and
// the private network never came up. Found is what the machine runs.
@@ -738,7 +738,7 @@ func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) {
}
}
func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
func caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
// No unit before the apply: nothing of a predecessor's to hold.
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}}
@@ -750,7 +750,7 @@ func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) {
}
}
func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
func caseAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "predecessor-data")
if err := os.Mkdir(data, 0o700); err != nil {
@@ -788,7 +788,7 @@ func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) {
}
}
func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
func caseAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
dir := t.TempDir()
data := filepath.Join(dir, "data")
m := &machine{containers: map[string]*fakeContainer{}}
@@ -801,7 +801,7 @@ func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) {
}
}
func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
func caseARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
dir, page, m := predecessor(t)
step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `",
"run-once":true,"network":"container:hello-web"}`
@@ -818,7 +818,7 @@ func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) {
}
}
func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
func caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
// An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the
// host holds one anyway, since what it would run in is the predecessor's.
dir, page, m := predecessor(t)
@@ -850,7 +850,7 @@ func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) {
const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000"
func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
func caseAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
dir := t.TempDir()
at := filepath.Join(dir, "site")
if err := os.Mkdir(at, 0o750); err != nil {
@@ -874,7 +874,7 @@ func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) {
}
}
func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
func caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
dir := t.TempDir()
was := unitDir
unitDir = dir
@@ -899,7 +899,7 @@ func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) {
}
}
func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
func caseAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
dir := t.TempDir()
m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}}
report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"),
@@ -915,7 +915,7 @@ func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) {
}
}
func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
func caseMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
// The runtime's socket, the kernel's filesystems and the clock are on every machine; a
// container mounting them shares nothing a predecessor kept (novox/hq ADR 0103).
dir := t.TempDir()
@@ -950,7 +950,7 @@ func (u *unreachable) run(_ context.Context, name string, args ...string) (strin
return "", nil
}
func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
func caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
dir := t.TempDir()
u := &unreachable{}
d := adopted(t, untaken("hello-web.server"),
@@ -970,7 +970,7 @@ func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *te
}
}
func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
func caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
dir, page, m := predecessor(t)
d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
_, state := applyAdopted(t, d, store.State{}, m, dir)
@@ -988,7 +988,7 @@ func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
}
}
func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
func caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
dir := t.TempDir()
run := func(_ context.Context, name string, args ...string) (string, error) {
args = oldShape(name, args)
@@ -1014,7 +1014,7 @@ func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
}
}
func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
func caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
// A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put
// back a service somebody took down on purpose (novox/hq ADR 0103).
dir := t.TempDir()
@@ -1035,7 +1035,7 @@ func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) {
}
}
func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
func caseAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
// The predecessor's own service from a package, running: not the mesh's to restart.
dir := t.TempDir()
conf := filepath.Join(dir, "hello.conf")
@@ -1054,7 +1054,7 @@ func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) {
}
}
func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
func caseASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
// Held, let go when the module was unassigned, rewritten by the predecessor, found again:
// both originals are kept, and the report names the one it kept (novox/hq ADR 0100).
dir := t.TempDir()
@@ -1083,7 +1083,7 @@ func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) {
}
}
func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
func caseAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
dir, page, m := predecessor(t)
_, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)),
store.State{}, m, dir)
@@ -1111,7 +1111,7 @@ func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) {
}
}
func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
func caseAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
// An access record says the mesh set permissions on a path it does not own — which is what it
// does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103).
dir := t.TempDir()
@@ -1144,3 +1144,51 @@ func asideShape(args []string) []string {
}
return args
}
// TestHold runs the cases of this file as subtests of one test (the test budget: one test, a table of cases).
func TestHold(t *testing.T) {
for _, c := range []struct {
name string
run func(*testing.T)
}{
{"AFoundFileOfAnUntakenModuleIsKeptAsItIs", caseAFoundFileOfAnUntakenModuleIsKeptAsItIs},
{"AFoundContainerOfAnUntakenModuleIsNotReplaced", caseAFoundContainerOfAnUntakenModuleIsNotReplaced},
{"WhatIsNotFoundIsCreatedWhenAssigned", caseWhatIsNotFoundIsCreatedWhenAssigned},
{"AFileThisHostWroteIsNotFound", caseAFileThisHostWroteIsNotFound},
{"AContainerAHostMadeIsNotFound", caseAContainerAHostMadeIsNotFound},
{"TheGenesisStoreAdoptedInPlaceIsNotFound", caseTheGenesisStoreAdoptedInPlaceIsNotFound},
{"TakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives", caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives},
{"AHeldFileIsNeverRemovedWhenItsModuleIsUnassigned", caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned},
{"AHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns", caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns},
{"ACarriedApplyLetsNoHoldGo", caseACarriedApplyLetsNoHoldGo},
{"AHeldFileRewrittenIsReportedAndNotReverted", caseAHeldFileRewrittenIsReportedAndNotReverted},
{"AHeldContainerStoppedOrReplacedIsReportedAndNotRestarted", caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted},
{"AHeldFileThatVanishesIsNotCreated", caseAHeldFileThatVanishesIsNotCreated},
{"AConvergedNodeStillReplacesWhatItFinds", caseAConvergedNodeStillReplacesWhatItFinds},
{"AFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt", caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt},
{"AFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents", caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents},
{"ADirectoryMadeInTheSameApplyIsNotFound", caseADirectoryMadeInTheSameApplyIsNotFound},
{"AFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted", caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted},
{"AHeldServiceIsStillReloadedButNeverRestarted", caseAHeldServiceIsStillReloadedButNeverRestarted},
{"AUnitAPackageOnlyShipsIsNotFound", caseAUnitAPackageOnlyShipsIsNotFound},
{"AServiceWhoseUnitIsNotThereIsAppliedAsUsual", caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual},
{"AContainerThatWouldMountFoundDataIsNotCreated", caseAContainerThatWouldMountFoundDataIsNotCreated},
{"AContainerMountingWhatTheMeshMadeIsCreated", caseAContainerMountingWhatTheMeshMadeIsCreated},
{"ARunOnceStepInAHeldContainerIsHeld", caseARunOnceStepInAHeldContainerIsHeld},
{"AnActionInAHeldContainerIsHeldUntilItsModuleIsTaken", caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken},
{"AnArchiveOverSomethingFoundIsNotUnpacked", caseAnArchiveOverSomethingFoundIsNotUnpacked},
{"AProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted", caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted},
{"AUserFoundOnTheMachineKeepsItsShellAndGroups", caseAUserFoundOnTheMachineKeepsItsShellAndGroups},
{"MountingTheMachinesOwnPlumbingIsNotFoundData", caseMountingTheMachinesOwnPlumbingIsNotFoundData},
{"ARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer", caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer},
{"AHeldContainerStaysHeldWhenTheRuntimeCannotAnswer", caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer},
{"AVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer", caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer},
{"AUnitSomebodyInstalledIsHeldWhateverStateItIsIn", caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn},
{"APackagedUnitTheMachineUsesIsStillHeld", caseAPackagedUnitTheMachineUsesIsStillHeld},
{"ASecondOriginalAtTheSamePathIsKeptToo", caseASecondOriginalAtTheSamePathIsKeptToo},
{"AHoldLetGoAndFoundAgainKeepsBothOriginals", caseAHoldLetGoAndFoundAgainKeepsBothOriginals},
{"APathTheMeshOnlySetAccessOnIsStillFound", caseAPathTheMeshOnlySetAccessOnIsStillFound},
} {
t.Run(c.name, c.run)
}
}
internal/apply/kept_artifact.go
+29
View File
@@ -0,0 +1,29 @@
package apply
import "context"
// The build the artifact store's holder keeps on a build node (novox/hq design 47, *The artifact store's holder takes its
// own new build even when its seat is the one broken*): the holder's program cannot be fetched from the store it serves, so
// its done build names the node that keeps it and the node-engine fetches it from there, through the build seat's verb.
// How is the caller's (a hook on the apply's context, as the files' link is); what apply does is use the bytes in place of
// the source's fetch, check them against the declared digest as every fetch is, and gate the module when they cannot be had.
// KeptFetch gives the bytes of the artifact a module's done build kept on a build node, by the artifact's digest
// ("sha256:" and 64 hex digits, as a declaration names it). kept is false when the module's build kept no such artifact:
// the artifact is then fetched from its source, as before. An error says the artifact is kept and could not be had.
type KeptFetch func(ctx context.Context, module, digest string) (body []byte, kept bool, err error)
type keptFetchKey struct{}
// WithKept is ctx carrying how a kept artifact is fetched.
func WithKept(ctx context.Context, f KeptFetch) context.Context {
if f == nil {
return ctx
}
return context.WithValue(ctx, keptFetchKey{}, f)
}
func keptFetchFrom(ctx context.Context) KeptFetch {
f, _ := ctx.Value(keptFetchKey{}).(KeptFetch)
return f
}
internal/apply/placement_guard.go
+29 -4
View File
@@ -54,6 +54,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
@@ -303,8 +304,28 @@ func splitQuoted(s string) []string {
return out
}
// engineModule is the module whose resources may place in the engine's own trees.
const engineModule = "mesh-host"
// engineModuleOf answers the module whose resources may place in the engine's own trees: the module whose entry of the
// catalog half claims the seat node-engine (design 47 G §527), asked of the node's composer; none until one is given.
var engineModuleOf atomic.Pointer[func() string]
// UseEngineModule gives the apply guard the function that answers the node-engine's own module; nil takes it away. The
// guard fixes no name (§364, §527).
func UseEngineModule(f func() string) {
if f == nil {
engineModuleOf.Store(nil)
return
}
engineModuleOf.Store(&f)
}
// isEngineModule is whether module is the node-engine's own.
func isEngineModule(module string) bool {
f := engineModuleOf.Load()
if f == nil || module == "" {
return false
}
return (*f)() == module
}
// passwdFile is the user database homes are read from. A variable so a test names its own.
var passwdFile = "/etc/passwd"
@@ -365,11 +386,15 @@ func below(path, dir string) bool {
// atOrBelow says whether path is dir or below it.
func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) }
// evalSymlinks follows the links of an existing path; a variable so a test judges paths whose machine it does not touch (a mount
// of the machine it runs on may not answer).
var evalSymlinks = filepath.EvalSymlinks
// resolved is a path with every link in it followed, as far as the path exists, and the rest as declared.
func resolved(path string) string {
rest := ""
for p := path; ; p = filepath.Dir(p) {
if real, err := filepath.EvalSymlinks(p); err == nil {
if real, err := evalSymlinks(p); err == nil {
return filepath.Clean(filepath.Join(real, rest))
}
if filepath.Dir(p) == p {
@@ -531,7 +556,7 @@ func refuseOne(path string, kind placing, module, owner string, declared map[str
return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree}
}
}
if module != engineModule {
if !isEngineModule(module) {
for _, tree := range engineTrees {
if atOrBelow(path, tree) {
return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " +
internal/apply/placement_guard_test.go
+25
View File
@@ -178,6 +178,11 @@ func TestAFoundDirectoryIsUsedAsFound(t *testing.T) {
// The module's own directories keep being placed: below /var/lib, below /etc as root's, a container's mount of
// the machine's docker socket.
func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) {
// The paths below are judged as written: no link of them is followed on the machine the test runs on, whose mounts of them
// (a share that does not answer) it does not depend on.
was := evalSymlinks
evalSymlinks = func(p string) (string, error) { return p, nil }
t.Cleanup(func() { evalSymlinks = was })
for _, path := range []string{"/var/lib/notes/data", "/etc/systemd/logind.conf.d", "/var/run/fail2ban",
"/srv/notes", "/mnt/plex/config", "/storage/media", "/var/lib/mesh/registry"} {
if err := refusePath(path, placingDirectory, "notes", ""); err != nil {
@@ -193,4 +198,24 @@ func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) {
if err := refusePath("/var/lib/notes/data", placingDirectory, "notes", "1001:2000"); err != nil {
t.Errorf("an owner below /var/lib: %v", err)
}
// Design 47 G §364, §527: the node-engine's own trees are placed in by the module the composer says claims its seat,
// whatever that module is called, and by no other; with none said, by nobody.
tree := engineTrees[0]
t.Cleanup(func() { UseEngineModule(nil) })
for _, c := range []struct {
engine func() string
module string
placed bool
}{
{func() string { return "the-engine" }, "the-engine", true},
{func() string { return "the-engine" }, "notes", false},
{func() string { return "the-engine" }, "", false},
{func() string { return "" }, "", false},
{nil, "the-engine", false},
} {
UseEngineModule(c.engine)
if err := refusePath(tree, placingDirectory, c.module, ""); (err == nil) != c.placed {
t.Errorf("%s for module %q: %v, want placed %v", tree, c.module, err, c.placed)
}
}
}
internal/apply/process.go
+19
View File
@@ -274,6 +274,25 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
return out, nil
}
// NodeEnv is the variable every process the node-engine runs reads its node's name from (design 47 G A-342), the one
// mesh-sdk reads (MESH_NODE).
const NodeEnv = "MESH_NODE"
// withNode is the process as it runs on node: its own env with NodeEnv beside it (design 47 G A-342). The validator
// refuses a resource that sets it, so the mesh's value is the only one; with no node named the process is as declared.
func withNode(r *declaration.Process, node string) *declaration.Process {
if node == "" {
return r
}
p := *r
p.Env = make(map[string]string, len(r.Env)+1)
for k, v := range r.Env {
p.Env[k] = v
}
p.Env[NodeEnv] = node
return &p
}
// unitFor is the unit the mesh writes for a daemon.
//
// **Generated whole and never edited in place**, the same rule as every other managed file: an
internal/apply/process_test.go
+7
View File
@@ -319,11 +319,17 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
}
d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`)
d.For = "anchor"
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
// Design 47 G A-342: the process runs with its node's name as MESH_NODE beside its own env.
if b, err := os.ReadFile(filepath.Join(units, "node-tools.service")); err != nil || !strings.Contains(string(b), `Environment="MESH_NODE=anchor"`) ||
!strings.Contains(string(b), `Environment="MESH_TOOL_MODULES=`) {
t.Errorf("the unit does not carry the node's name beside its env (%v):\n%s", err, b)
}
if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" {
t.Fatalf("first apply: %+v, want created", o)
}
@@ -360,6 +366,7 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) {
// An input of its unit that differs restarts it, and the line says which (issue 1966).
changed := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+
`","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js,b=/y/index.js"}}`)
changed.For = "anchor"
fourth, _, err := Apply(context.Background(), archHost(t), changed, state, store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
internal/apply/replace_image_test.go
+41
View File
@@ -164,6 +164,47 @@ func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testi
t.Errorf("the container's outcome is %+v, want skipped", o)
}
})
// Design 47 (W13.8): the holder's own program is fetched from the build node that kept it, not from the store; when it
// cannot be had, or is not the declared digest, nothing of the module is replaced and the failure says so.
for _, row := range []struct {
name string
fetch KeptFetch
wantErr string
}{
{"the node that keeps it is away", func(context.Context, string, string) ([]byte, bool, error) {
return nil, true, errors.New("novox keeps it and does not answer")
}, "novox keeps it and does not answer"},
{"what the node kept is not the declared digest", func(context.Context, string, string) ([]byte, bool, error) {
return []byte("other"), true, nil
}, "what arrived is"},
} {
t.Run("a kept artifact: "+row.name, func(t *testing.T) {
gone := httptest.NewServer(http.NotFoundHandler())
source := gone.URL + "/program.tar.gz"
gone.Close()
was := storeAwayAtMost
storeAwayAtMost = 0
t.Cleanup(func() { storeAwayAtMost = was })
d := parseTrusted(t, `{"declaration":1,"resources":[
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
]}`)
var asked []string
fetch := func(ctx context.Context, module, digest string) ([]byte, bool, error) {
asked = append(asked, module+" "+digest)
return row.fetch(ctx, module, digest)
}
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
_, _, err := Apply(WithKept(context.Background(), fetch), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
if err == nil || !strings.Contains(err.Error(), row.wantErr) {
t.Errorf("the failure is %v, want it to say %q", err, row.wantErr)
}
if len(asked) != 1 || asked[0] != "distribution sha256:"+strings.Repeat("0", 64) {
t.Errorf("the kept fetch was asked %q", asked)
}
})
}
}
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
internal/apply/stop.go
+69
View File
@@ -0,0 +1,69 @@
package apply
import (
"context"
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// StopModule stops the units and containers the node's record holds for a module and removes nothing: no unit, file,
// container, data or record, so what the module kept survives and a later Apply starts the build it is given (design 47
// G A-98, §297). Units come first, a timer before the service it starts, then containers; a unit the mesh never held
// (Stateless) is left as it is. It answers what it stopped, and the first failure.
func StopModule(ctx context.Context, sys system.System, known store.State, module string, run Runner) ([]string, error) {
var timers, services, containers []store.Applied
for _, r := range known.Resources {
if store.OwnerOf(r, declaration.ModuleGuess) != module {
continue
}
switch r.Type {
case string(declaration.TypeService):
if r.Stateless {
continue
}
if strings.HasSuffix(r.Target, ".timer") {
timers = append(timers, r)
} else {
services = append(services, r)
}
case string(declaration.TypeContainer):
containers = append(containers, r)
}
}
var stopped []string
for _, r := range append(timers, services...) {
manager := managerFor(r.Scope, r.User, run)
if state, err := sys.ServiceState(ctx, manager, r.Target); err != nil {
if strings.Contains(err.Error(), "does not exist on this machine") {
continue
}
return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err)
} else if state == "stopped" {
continue
}
if err := sys.SetServiceState(ctx, manager, r.Target, "stopped"); err != nil {
return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err)
}
stopped = append(stopped, r.Target)
}
if len(containers) == 0 {
return stopped, nil
}
cri, err := containerRuntime(ctx, run)
if err != nil {
return stopped, err
}
sort.Slice(containers, func(i, j int) bool { return containers[i].Target < containers[j].Target })
for _, r := range containers {
if _, err := run(ctx, cri, "stop", r.Target); err != nil {
return stopped, fmt.Errorf("stopping the container %s of %s: %w", r.Target, module, err)
}
stopped = append(stopped, r.Target)
}
return stopped, nil
}
internal/apply/stop_test.go
+45
View File
@@ -0,0 +1,45 @@
package apply
import (
"context"
"reflect"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Stop stops a module's timer, then its service, then its container, and leaves every record, and another module's
// units, as they are (design 47 G A-98).
func TestStopModuleStopsAModulesUnitsAndContainersAndRemovesNothing(t *testing.T) {
m := unitsMachine(map[string]*fakeUnit{
"app.service": {active: "active", enabled: "enabled"},
"app.timer": {active: "active", enabled: "enabled"},
"other.service": {active: "active", enabled: "enabled"},
})
state := store.State{}
for _, r := range []store.Applied{
{ID: "app.run", Type: "service", Target: "app.service"},
{ID: "app.tick", Type: "service", Target: "app.timer"},
{ID: "app.c", Type: "container", Target: "app-box"},
{ID: "other.run", Type: "service", Target: "other.service"},
{ID: "other.c", Type: "container", Target: "other-box"},
} {
state.Record(r)
}
got, err := StopModule(context.Background(), archHost(t), state, "app", m.run)
if err != nil {
t.Fatal(err)
}
if want := []string{"app.timer", "app.service", "app-box"}; !reflect.DeepEqual(got, want) {
t.Fatalf("stopped %v, want %v, the timer first", got, want)
}
if m.units["app.service"].active != "inactive" || m.units["app.timer"].active != "inactive" || m.units["other.service"].active != "active" {
t.Fatalf("units are %+v", m.units)
}
if !m.did("docker stop app-box") || m.did("docker stop other-box") || m.did("docker rm") {
t.Fatalf("the machine was asked %v", m.asked)
}
if len(state.Resources) != 5 {
t.Fatalf("a record was removed: %d left", len(state.Resources))
}
}
internal/buckets/buckets.go
+2 -2
View File
@@ -9,8 +9,8 @@ import (
vault "git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault"
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
)
internal/buckets/bucketstest/bucketstest.go
+3 -3
View File
@@ -4,11 +4,11 @@ package bucketstest
import (
"encoding/json"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Version is the value version the holders write.
internal/compose/composed.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
engcomp "github.com/novox/mesh-host/internal/compositions"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
@@ -388,8 +388,8 @@ func ToolsRunning(running []engcomp.Composed) map[string][]catalogue.Bundle {
}
var own map[string]string
for w, v := range words[module] {
if w == catalogue.FilesOwnedPaths || w == catalogue.ReadsSecretPlaces {
continue // the mesh's to say, composed again
if w == catalogue.FilesOwnedPaths || w == catalogue.ReadsSecretPlaces || w == engcomp.FingerprintVar {
continue // the mesh's to say, composed again (the fingerprint is the want's, §519)
}
if own == nil {
own = map[string]string{}
internal/compose/composed_test.go
+7 -2
View File
@@ -2,7 +2,7 @@ package compose
import (
"encoding/json"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
engcomp "github.com/novox/mesh-host/internal/compositions"
"sort"
"strings"
@@ -180,7 +180,12 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) {
}
}
}
if env[catalogue.RuntimeToolModules] != served || env[catalogue.RuntimeToolEnv] != `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf"}}` {
// §519: the words each bundle's module is given hold the fingerprint of the want it applies, where there is one.
words := `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf"}}`
if fp := row.fps["lamp"]; fp != "" {
words = `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf","MESH_FINGERPRINT":"` + fp + `"}}`
}
if env[catalogue.RuntimeToolModules] != served || env[catalogue.RuntimeToolEnv] != words {
t.Errorf("%s: the runtime serves %v with %v, want what runs of lamp, %s", row.name,
env[catalogue.RuntimeToolModules], env[catalogue.RuntimeToolEnv], served)
}
internal/compose/frombus/answer_rows_test.go
+2 -2
View File
@@ -8,9 +8,9 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
engcomp "github.com/novox/mesh-host/internal/compositions"
"os"
"reflect"
internal/compose/frombus/frombus.go
+73 -3
View File
@@ -45,9 +45,9 @@ import (
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
engcomp "github.com/novox/mesh-host/internal/compositions"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
@@ -81,7 +81,7 @@ type Reader interface {
// and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key
// the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it.
Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error)
// Build is a module's build at a fingerprint, the builder's key (builds.BuilderBuildsBucket, BuilderBuildKey); nil
// Build is a module's build at a fingerprint, the builder's key (BuilderBuildKey); nil
// when the builder holds none. The composing node composes each of its modules from its build (issue 1126).
Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error)
// Archive is the archive at url, a kept reference routed to the artifact store as the composing node reaches it
@@ -156,6 +156,17 @@ type Scope struct {
Assignments map[string]*meshmoduleassigner.Assignments
InOwn []string
Running []engcomp.Composed
// Heard is, by node and module, what each node said it runs, from the node-engine seat's event `running` kept by the
// composing node's own consumer (design 47 G §199): the passing of a seat reads it, and a module nothing was heard of
// counts by the trunk's entry. Never read from a have key, a want or a call.
Heard map[string]map[string]Heard
// Engine is told, at each composing of the node's input, the node-engine's own module: the one whose entry of the catalog half
// claims the seat node-engine, else the one of the node's heard modules that runs the build OwnFingerprint names (design 47 G
// §527); "" when neither shows it. Nil: told to nobody.
Engine func(module string)
// OwnFingerprint is the fingerprint this node-engine was started at (its MESH_FINGERPRINT); empty when it was started
// without one, as an engine of before G.
OwnFingerprint string
// HeldToday is, in an answer, the modules the node keeps today as they run ("node" for its own composition), each with
// how ("held", "failed", "runs a build of the old form"): a candidate that cannot compose one, or holds one only
// behind one of them, keeps it as today, which is said under CandidateAnswer.Held and is no refusal (issue 1126).
@@ -551,6 +562,7 @@ func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Conte
if err := fromBuilds(ctx, r, build, &in, s, b.assignments[s.Node], b.modules, read, parse, refuse, wanted); err != nil {
return compose.Input{}, err
}
keepHeard(&in, s, read)
// Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5).
// Asked only when a module placed on it declares one: a node none of whose modules has a secret composes
// whether or not the vault answers.
@@ -581,6 +593,61 @@ func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Conte
return in, nil
}
// Heard is one `running` event as the scope keeps it: the fingerprint of the build, the digest of its manifest, or Stopped.
type Heard struct {
Fingerprint string
Manifest string
Stopped bool
}
// EngineModule is the node-engine's own module (design 47 G §527): the module whose catalog entry claims the seat node-engine,
// which the validator makes one per revision (§508). With none claiming it, which only a revision of before G shows, as at a
// revert of G, the module this node heard run the build its own fingerprint names; "" when neither shows it.
func EngineModule(in compose.Input, node, ownFingerprint string) string {
for _, name := range sortedKeys(in.Catalogue) {
if resolution.ClaimsEngine(in.Catalogue[name]) {
return name
}
}
if ownFingerprint == "" {
return ""
}
for i := range in.Nodes {
if in.Nodes[i].Name != node {
continue
}
for _, module := range sortedKeys(in.Nodes[i].Heard) {
if in.Nodes[i].Heard[module].Fingerprint == ownFingerprint {
return module
}
}
}
return ""
}
// keepHeard puts, as each node's Heard, what it said it runs of each module it is assigned, with the manifest of the build
// read by the event's digest; a digest that cannot be read leaves the entry unread, and the passing says so (§459).
func keepHeard(in *compose.Input, s Scope, read func(module, digest string) (catalogue.Manifest, error)) {
for i := range in.Nodes {
n := &in.Nodes[i]
heard := s.Heard[n.Name]
if len(heard) == 0 {
continue
}
n.Heard = map[string]compose.Heard{}
for _, module := range sortedKeys(heard) {
h := heard[module]
got := compose.Heard{Fingerprint: h.Fingerprint, Stopped: h.Stopped}
if !h.Stopped && h.Manifest != "" {
if man, err := read(module, h.Manifest); err == nil {
got.Manifest, got.Read = man, true
}
}
n.Heard[module] = got
}
}
}
// Compose is the node's compositions over the bus (ADR 0547 §2): input for s, then composed at the fingerprints fps
// gives each module in the Reading mode, as the controller composes them, by the one rule its answer to a candidate
// follows too (composeEach, issue 1126): a module refused alone (its build unread or not taken in, its composition
@@ -608,6 +675,9 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s
if err != nil {
return nil, nil, held, err
}
if s.Engine != nil {
s.Engine(EngineModule(in, s.Node, s.OwnFingerprint))
}
if s.Holds != nil {
holds := map[string]compose.Hold{}
for name := range held {
internal/compose/frombus/frombus_test.go
+33 -2
View File
@@ -15,10 +15,11 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"github.com/novox/mesh-host/internal/compositions"
"maps"
"os"
"reflect"
"slices"
@@ -575,6 +576,36 @@ func TestInput(t *testing.T) {
t.Errorf("the want's build is kept on the composing node alone: %v, %v", in.Nodes, err)
}
// Design 47 G §199 (test 56): what each node said it runs (the seat node-engine's event running) is that node's Heard,
// with the manifest read by the event's digest; a digest that cannot be read leaves the entry unread, and a stopped
// module has none.
in, err = Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Heard: map[string]map[string]Heard{
"laptop": {"xterm": {Fingerprint: "f1", Manifest: digest}, "media": {Stopped: true}}, "anchor": {"xterm": {Fingerprint: "f2", Manifest: "gone"}}}})
if err != nil {
t.Fatal(err)
}
for _, n := range in.Nodes {
h := n.Heard["xterm"]
switch {
case n.Name == "laptop" && (!h.Read || h.Fingerprint != "f1" || h.Manifest.Module != "xterm" || !n.Heard["media"].Stopped):
t.Errorf("laptop's Heard holds %+v", n.Heard)
case n.Name == "anchor" && (h.Read || h.Fingerprint != "f2"):
t.Errorf("anchor's Heard holds %+v, want f2 unread", n.Heard)
}
}
// Design 47 G §527: the node-engine's own module is the one whose entry claims its seat; with none, the one this node heard
// run the build its own fingerprint names; with neither, none.
if got := EngineModule(in, "laptop", "f1"); got != "xterm" {
t.Errorf("by the own fingerprint: %q, want xterm", got)
}
if got := EngineModule(in, "laptop", "other"); got != "" {
t.Errorf("an unheard fingerprint names %q", got)
}
in.Catalogue = maps.Clone(in.Catalogue)
in.Catalogue["the-engine"] = catalogue.Manifest{Module: "the-engine", Claims: []catalogue.Claim{{Name: "node-engine"}}}
if got := EngineModule(in, "laptop", "f1"); got != "the-engine" {
t.Errorf("by the claim: %q, want the-engine", got)
}
// Issue 1126, the seed's refusals. restic and the provisioner's client as the catalogue holds them (mesh-catalog
// c05aa5e6), restic moved with an own secret the vault leaves out (it hands it to the client, mesh-vault ownnode.go
// handed): composed from the indexer's manifest restic is not handed and stands in for that secret, as novox's,
internal/compose/frombus/frombustest/frombustest.go
+2 -2
View File
@@ -14,9 +14,9 @@ import (
"errors"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"github.com/novox/mesh-host/internal/compose"
"github.com/novox/mesh-host/internal/compose/frombus"
"strings"
internal/compose/frombus/frombustest/frombustest_test.go
+2 -2
View File
@@ -7,9 +7,9 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
engcomp "github.com/novox/mesh-host/internal/compositions"
"reflect"
"slices"
internal/compose/frombus/held_rows_test.go
+2 -2
View File
@@ -7,9 +7,9 @@ import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"github.com/novox/mesh-host/internal/compose"
"github.com/novox/mesh-host/internal/compose/resolution"
engcomp "github.com/novox/mesh-host/internal/compositions"
internal/compose/frombus/states.go
+2 -2
View File
@@ -19,9 +19,9 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/compose"
"github.com/novox/mesh-host/internal/compose/resolution"
internal/compose/input.go
+12
View File
@@ -240,6 +240,9 @@ type Node struct {
// own resolution only, never to another node's, and a declaration carries the kept commit for the module.
Kept map[string]catalogue.Manifest
KeptBuilds map[string]string
// Heard is, by module, what this node said it runs, from the node-engine seat's event `running` (design 47 G §199): the
// passing of a seat resolves the build each assignment runs from it. Absent for a module nothing was heard of.
Heard map[string]Heard
// Contract is the contract of its node-engine's newest statement (inv.HealthOf); Stated is whether it ever stated one.
Contract int
Stated bool
@@ -437,6 +440,15 @@ type Credential struct {
Why string
}
// Heard is what a node's `running` event for a module said: the build's fingerprint, or Stopped (no fingerprint, §365), with
// the manifest of the build's entry read by the event's manifest digest (Read false when it could not be read).
type Heard struct {
Fingerprint string
Stopped bool
Manifest catalogue.Manifest
Read bool
}
// Run is the build an assignment names: the commit it is of and the build made from it (chain.Ref).
type Run struct {
Commit string
internal/compose/resolution/placement.go
+11 -2
View File
@@ -7,6 +7,7 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
)
@@ -56,8 +57,16 @@ func systemPath(path string) string {
return ""
}
// engineModule is the node-engine's own module, the one that places in engineTrees.
const engineModule = "mesh-host"
// ClaimsEngine is whether a manifest claims the seat node-engine: the node-engine's own module is the one whose entry of
// the catalog half does (design 47 G §527), never a name fixed in code.
func ClaimsEngine(m mcat.Manifest) bool {
for _, c := range m.Claims {
if c.Name == nodeengine.Seat {
return true
}
}
return false
}
// accessRef is how a module names one of its accesses: ${access:<id>}.
var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`)
internal/compose/resolution/runtime_test.go
+13 -4
View File
@@ -310,16 +310,25 @@ func TestAModuleOfItsOwnAccountIsServedByARuntimeOfItsOwn(t *testing.T) {
}
gob, img := builder("gob", "node-build-go"), builder("img", "node-build-image")
with.Needed["gob"], with.Needed["img"] = map[string]string{"broker": "own"}, map[string]string{"broker": "own"}
// The node-engine's module is the one whose entry claims its seat (design 47 G §527), whatever it is called; with none in
// the resolution its settings are not found and the default stands.
engine := mcat.Manifest{Module: "mesh-host", Claims: []mcat.Claim{{Name: "node-engine"}}}
for _, c := range []struct {
settings mcat.SettingsBy
share string
noEngine bool
}{
{nil, "50"},
{mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "75"},
{mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(0)}}}}, "50"},
{nil, "50", false},
{mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "75", false},
{mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(0)}}}}, "50", false},
{mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "50", true},
} {
with.Settings = c.settings
runtimes, err := resolutionOwnRuntimes(mcat.Resolution{Node: "anchor", Modules: []mcat.Manifest{gob, img, telegram, goRuntime}}, with)
modules := []mcat.Manifest{gob, img, telegram, goRuntime, engine}
if c.noEngine {
modules = modules[:4]
}
runtimes, err := resolutionOwnRuntimes(mcat.Resolution{Node: "anchor", Modules: modules}, with)
if err != nil {
t.Fatal(err)
}
internal/compose/resolution/seat_dependencies.go
+5 -2
View File
@@ -29,6 +29,10 @@ var appliedThrough = map[string]string{
// A variable rather than a constant only so a test can hold both behaviours; nothing else sets it.
var enforceSeatDependencies = true
// nodeEngineSeat is the seat the node-engine's own module claims (design 47 G §508, §527, §634): the
// foundation's host is the entry that claims it, never a module named here.
const nodeEngineSeat = "node-engine"
// foundationModules are the pieces genesis lays before any module exists (novox/hq ADR 0207 §5):
// the host and the private network. Registered as modules so they can be assigned, but what they
// declare is the installation's, not a module's, so it is never judged. The third piece, the
@@ -38,7 +42,6 @@ var enforceSeatDependencies = true
// The private network's module is overlay.Name, written out because the overlay package composes
// on top of this one; its resources are computed, which exempts it by the rule below as well.
var foundationModules = map[string]bool{
"mesh-host": true,
"mesh-wireguard": true,
}
@@ -46,7 +49,7 @@ var foundationModules = map[string]bool{
// module whose resources are computed is the mesh's by construction — the private network's peer
// list and its tools are the controller's, written per node — so it counts whatever its name.
func isFoundation(m mcat.Manifest) bool {
return foundationModules[m.Module] || m.Computed != ""
return foundationModules[m.Module] || m.ClaimsSeat(nodeEngineSeat) || m.Computed != ""
}
// DependsOn is every seat a module needs held on its node: derived from the resource types it
internal/compose/resolution/seat_dependencies_test.go
+14
View File
@@ -95,3 +95,17 @@ func TestWithTheSwitchFlippedAnUnmetDependencyRefusesTheNode(t *testing.T) {
t.Errorf("the first pass refused an unmet dependency: %v", err)
}
}
func TestTheFoundationHostIsTheEntryThatClaimsTheNodeEngineSeat(t *testing.T) {
svc := res("service", "x")
// Whatever its name, the entry claiming the seat is the foundation's: nothing to depend on.
engine := withResources(mod("the-engine", nil, nil, nil, mcat.Claim{Name: "node-engine"}), svc)
if got := DependsOn(nil, engine); len(got) != 0 {
t.Errorf("the entry claiming node-engine depends on %v, want nothing (§527)", got)
}
// A module named as the old code named the engine, claiming no such seat, is an ordinary module.
named := withResources(mod("mesh-host", nil, nil, nil), svc)
if got := DependsOn(nil, named); !reflect.DeepEqual(got, []string{mcat.ServiceManagerSeat}) {
t.Errorf("a module merely named mesh-host depends on %v, want the service manager seat", got)
}
}
internal/compose/resolution/yields.go
+3 -3
View File
@@ -21,14 +21,14 @@ func resolutionYieldsOf(r mcat.Resolution, m mcat.Manifest, with mcat.Rendering)
if !yields(m) {
return nil, nil
}
engine := mcat.Manifest{Module: engineModule}
var engine mcat.Manifest // the module claiming the seat node-engine; none in the resolution leaves the default
for _, e := range r.Modules {
if e.Module == engineModule {
if ClaimsEngine(e) {
engine = e
}
}
share := mcat.BuildCPUDefaultPercent
for _, s := range Effective(engine, with.Settings[engineModule]) {
for _, s := range Effective(engine, with.Settings[engine.Module]) {
if s.Key == mcat.BuildCPUShareSetting {
if n, ok := percentOf(s.Value); ok {
share = n
internal/compose/rest.go
+53 -24
View File
@@ -102,7 +102,7 @@ func (c *composer) theRestOfTheMeshNow(exclude string) (catalogue.World, Passes,
}
}
var passes Passes
holdings, passes = passSeatsOn(holdings, c.assignmentsOf(shelf, everyAssigned))
holdings, passes = passSeatsOn(holdings, c.assignmentsOf(everyAssigned), c.heard, c.entries(shelf))
for _, n := range nodes {
if n.Name == exclude {
continue
@@ -177,10 +177,11 @@ func (c *composer) theRestOfTheMeshNow(exclude string) (catalogue.World, Passes,
}
// passSeatsOn finds, in the composition that finds it, the new holder of each seat held once whose holder on record has left
// it and which has exactly one claimant able to hold it (catalogue.SeatsToPassOn, novox/hq ADR 0660), and answers the
// holdings as they are after it, with the passes. Nothing is recorded.
func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment) ([]catalogue.Held, Passes) {
passes, said := catalogue.SeatsToPassOn(holdings, assignments)
// it and which has exactly one claimant able to hold it (catalogue.SeatsToPassOn, novox/hq ADR 0660, design 47 G §199, §459), and
// answers the holdings as they are after it, with the passes. Nothing is recorded.
func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment, heard func(node, module string) (catalogue.Heard, bool),
entries catalogue.EntryReader) ([]catalogue.Held, Passes) {
passes, said := catalogue.SeatsToPassOn(holdings, assignments, heard, entries)
out := Passes{Said: said}
for _, p := range passes {
out.Passes = append(out.Passes, p)
@@ -197,11 +198,10 @@ func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment)
return holdings, out
}
// assignmentsOf are the assignments of every machine, as the passing of a seat reads them (novox/hq ADR 0660 clause 2): each
// with the manifest of the build its node runs where the Input holds it (the node's Kept: what it runs of a module kept at
// that build), else the trunk's on the shelf, which for the composing node's own modules is already the build they are
// wanted at. A module assigned and missing from both is still assigned, marked unread, so its holder is not taken for gone.
func (c *composer) assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment {
// assignmentsOf are the assignments of every machine, as the passing of a seat reads them (§459): a node, a module and no
// placement branch, this composer's view of the catalog half being the trunk's alone. The build each runs is resolved by
// catalogue.SeatsToPassOn from c.heard and c.entries.
func (c *composer) assignmentsOf(assigned map[string][]string) []catalogue.Assignment {
var out []catalogue.Assignment
names := make([]string, 0, len(assigned))
for node := range assigned {
@@ -209,26 +209,55 @@ func (c *composer) assignmentsOf(shelf map[string]catalogue.Manifest, assigned m
}
sort.Strings(names)
for _, node := range names {
var kept map[string]catalogue.Manifest
for _, n := range c.in.Nodes {
if n.Name == node {
kept = n.Kept
}
}
for _, module := range assigned[node] {
m, ok := kept[module]
if !ok {
m, ok = shelf[module]
}
if !ok {
m = catalogue.Manifest{Module: module}
}
out = append(out, catalogue.Assignment{Node: node, Manifest: m, Unread: !ok})
out = append(out, catalogue.Assignment{Node: node, Module: module})
}
}
return out
}
// heard is what the node-engine seat's `running` said of a module on a node (Node.Heard): a fingerprint or stopped. Nothing
// is heard of a module no event named, and the passing then takes the entry of the module on the trunk (§199).
func (c *composer) heard(node, module string) (catalogue.Heard, bool) {
for _, n := range c.in.Nodes {
if n.Name == node {
h, ok := n.Heard[module]
return catalogue.Heard{Fingerprint: h.Fingerprint, Stopped: h.Stopped}, ok
}
}
return catalogue.Heard{}, false
}
// entries are the composer's reader of the catalog half for the passing of a seat: an entry by the fingerprint a node said it
// runs, read through the digest its event named, and a module's entry on the trunk, the shelf.
func (c *composer) entries(shelf map[string]catalogue.Manifest) catalogue.EntryReader {
byFingerprint := map[string]catalogue.Manifest{}
for _, n := range c.in.Nodes {
for _, h := range n.Heard {
if h.Read && h.Fingerprint != "" {
byFingerprint[h.Fingerprint] = h.Manifest
}
}
}
return entryReader{byFingerprint: byFingerprint, shelf: shelf}
}
type entryReader struct {
byFingerprint map[string]catalogue.Manifest
shelf map[string]catalogue.Manifest
}
func (r entryReader) ByFingerprint(fingerprint string) (catalogue.Manifest, bool) {
m, ok := r.byFingerprint[fingerprint]
return m, ok
}
// OnBranch answers the trunk's entry whatever the branch: this composer's shelf is the trunk's alone.
func (r entryReader) OnBranch(module, _ string) (catalogue.Manifest, bool) {
m, ok := r.shelf[module]
return m, ok
}
// Lines are what the passes say, one each, then what the finding itself says: a pass is composed so and recorded by nobody
// (ADR 0660 clause 4).
func (p Passes) Lines() []string {
internal/compose/rest_test.go
+30 -9
View File
@@ -162,19 +162,18 @@ func TestASeatWhoseHolderIsAssignedNowhereGoesToItsOnlyClaimant(t *testing.T) {
func TestNoSeatIsPassedForTwoClaimantsAnAssignedHolderOrNoClaimant(t *testing.T) {
withSeat(t)
held := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
_, passes := passSeatsOn(held, []catalogue.Assignment{
{Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}})
shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate"), "other": aBrokerClaiming("other"), "old-broker": aBrokerClaiming("old-broker")}
_, passes := passSeatsOn(held, []catalogue.Assignment{{Node: "novox", Module: "gate"}, {Node: "ace", Module: "other"}}, nil, entryReader{shelf: shelf})
said := strings.Join(passes.Said, "\n")
if len(passes.Passes) != 0 || !strings.Contains(said, "gate") || !strings.Contains(said, "other") ||
!strings.Contains(said, "seat mesh-broker --to") {
t.Fatalf("passes %+v said %q", passes.Passes, said)
}
got, passes := passSeatsOn(held, []catalogue.Assignment{
{Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}})
got, passes := passSeatsOn(held, []catalogue.Assignment{{Node: "novox", Module: "old-broker"}, {Node: "novox", Module: "gate"}}, nil, entryReader{shelf: shelf})
if len(passes.Passes) != 0 || len(got) != 1 || got[0].Module != "old-broker" {
t.Fatalf("holder assigned: %+v %+v", got, passes)
}
got, passes = passSeatsOn(held, nil)
got, passes = passSeatsOn(held, nil, nil, entryReader{shelf: shelf})
if len(passes.Passes) != 0 || len(got) != 1 || got[0].Module != "old-broker" {
t.Fatalf("no claimant: %+v %+v", got, passes)
}
@@ -183,13 +182,35 @@ func TestNoSeatIsPassedForTwoClaimantsAnAssignedHolderOrNoClaimant(t *testing.T)
// seat_pass_test.go TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone.
func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) {
withSeat(t)
as := (&composer{}).assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")},
map[string][]string{"novox": {"gate", "old-broker"}})
as := (&composer{}).assignmentsOf(map[string][]string{"novox": {"gate", "old-broker"}})
passes, _ := catalogue.SeatsToPassOn(
[]catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as)
[]catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as, nil,
entryReader{shelf: map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}})
if len(passes) != 0 {
t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes)
}
// Design 47 G §199 (test 56): the build a node runs is the one it said in its running event, the trunk's entry counting
// only for a node that said nothing. The trunk's old-broker still claims the seat; the one the away node runs does not.
for name, c := range map[string]struct {
heard Heard
passTo string
}{
"the away node runs a build that no longer claims the seat": {Heard{Fingerprint: "f1", Manifest: catalogue.Manifest{Module: "old-broker"}, Read: true}, "gate"},
"the away node says it stopped it": {Heard{Stopped: true}, "gate"},
"the away node's build cannot be read": {Heard{Fingerprint: "f1"}, ""},
"the away node runs a build that claims the seat": {Heard{Fingerprint: "f1", Manifest: aBrokerClaiming("old-broker"), Read: true}, ""},
} {
in := Input{
Catalogue: map[string]catalogue.Manifest{"old-broker": aBrokerClaiming("old-broker"), "gate": aBrokerClaiming("gate")},
Nodes: []Node{{Name: "away", Assigned: []string{"old-broker"}, Heard: map[string]Heard{"old-broker": c.heard}},
{Name: "novox", Assigned: []string{"gate"}}},
Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "away", Module: "old-broker"}},
}
_, got, _, err := TheRestOfTheMesh(in, "")
if err != nil || (c.passTo == "") != (len(got.Passes) == 0) || (c.passTo != "" && got.Passes[0].To.Module != c.passTo) {
t.Errorf("%s: %v %+v", name, err, got)
}
}
}
// The not-resolving node of a view with a holder on record: holdings reach the first pass (novox/hq ADR 0131), so a node running
@@ -267,7 +288,7 @@ func TestAHolderWhoseManifestDroppedTheSeatGoesToItsOnlyClaimant(t *testing.T) {
}
// The build the holder's node runs still claims the seat, though the trunk dropped it: the holder keeps it (ADR 0660 §2).
running := mk("gate")
running.Nodes[0].Kept = map[string]catalogue.Manifest{"old-broker": aBrokerClaiming("old-broker")}
running.Nodes[0].Heard = map[string]Heard{"old-broker": {Fingerprint: "f", Manifest: aBrokerClaiming("old-broker"), Read: true}}
_, passes, _, err = TheRestOfTheMesh(running, "")
if err != nil || len(passes.Passes) != 0 || len(passes.Said) != 0 {
t.Fatalf("a holder whose running build claims the seat lost it: %v %+v", err, passes)
internal/composer/composer.go
+3 -3
View File
@@ -12,15 +12,15 @@ import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
"maps"
"slices"
"sort"
"strings"
sdk "git.novox.be/novox/mesh-delivery/sdk/composer"
sdk "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
internal/composer/composer_test.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
engcomp "github.com/novox/mesh-host/internal/compositions"
internal/compositions/compare_test.go
+67
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
seat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"slices"
"strings"
"testing"
@@ -165,4 +166,70 @@ func TestComparingCompositions(t *testing.T) {
if c := CompareComposed(map[string]Composed{"k": comp(file("conf", "a"))}, nil); len(c.Added) != 1 || c.Added[0].Module != "nats" || c.Added[0].Composed == "" {
t.Errorf("a composition not run is added: %+v", c)
}
// Design 47, *The fingerprint handover* (the composed resources are split here): a daemon or a container of a module
// is started with MESH_FINGERPRINT, the fingerprint of the want it applies, over any value its module declared of that
// name; a file, or a resource of no module, carries none.
split, err := Split("novox", []json.RawMessage{
json.RawMessage(`{"id":"d","type":"process","module":"nats","run":["x"],"env":{"A":"1","MESH_FINGERPRINT":"stale"}}`),
json.RawMessage(`{"id":"c","type":"container","module":"nats","image":"i"}`),
file("conf", "a"),
json.RawMessage(`{"id":"own","type":"process","run":["x"]}`),
}, map[string]string{"nats": fp}, nil, NodeOwn{})
if err != nil {
t.Fatal(err)
}
if len(split) != 2 {
t.Fatalf("split into %d compositions, want the module's and the node's own", len(split))
}
if n := len(split[mustKey(t, "novox", "nats", fp)].Resources); n != 3 {
t.Fatalf("the module's composition holds %d resources, want 3", n)
}
for _, raw := range split[mustKey(t, "novox", "nats", fp)].Resources {
var r struct {
ID string
Env map[string]string
}
_ = json.Unmarshal(raw, &r)
want := fp
if r.ID == "conf" {
want = ""
}
if r.Env[FingerprintVar] != want || (r.ID == "d" && r.Env["A"] != "1") {
t.Errorf("%s starts with %v, want %s=%q and its own words kept", r.ID, r.Env, FingerprintVar, want)
}
}
for _, raw := range split["novox"].Resources {
if strings.Contains(string(raw), FingerprintVar) {
t.Errorf("the node's own carries %s: %s", FingerprintVar, raw)
}
}
// §519, §618: the tool runner's process hands each bundle's module its own want's fingerprint in the words of its
// bundles, a module with bundles and no words included, and the runner's own fingerprint reaches no other module's.
fpLamp, fpBare := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
runner, err := Split("novox", []json.RawMessage{json.RawMessage(`{"id":"r","type":"process","module":"nats","run":["x"],"env":{` +
`"` + catalogue.RuntimeToolModules + `":"bare=/b/x,lamp=/l/y","` + catalogue.RuntimeToolEnv + `":"{\"lamp\":{\"LAMP_CONF\":\"/c\"}}"}}`)},
map[string]string{"nats": fp, "lamp": fpLamp, "bare": fpBare}, nil, NodeOwn{})
if err != nil {
t.Fatal(err)
}
var proc struct{ Env map[string]string }
_ = json.Unmarshal(runner[mustKey(t, "novox", "nats", fp)].Resources[0], &proc)
var given map[string]map[string]string
if err := json.Unmarshal([]byte(proc.Env[catalogue.RuntimeToolEnv]), &given); err != nil {
t.Fatal(err)
}
if given["lamp"][FingerprintVar] != fpLamp || given["lamp"]["LAMP_CONF"] != "/c" || given["bare"][FingerprintVar] != fpBare ||
len(given["nats"]) != 0 || proc.Env[FingerprintVar] != fp {
t.Errorf("the bundles' words are %v and the runner's env %v", given, proc.Env)
}
}
func mustKey(t *testing.T, node, module, fp string) string {
k, err := Key(node, module, fp)
if err != nil {
t.Fatal(err)
}
return k
}
internal/compositions/composed.go
+102 -2
View File
@@ -4,9 +4,11 @@ import (
"encoding/json"
"fmt"
"regexp"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -134,7 +136,7 @@ func Same(a, b Composed) bool {
// Key is the key of a module's composition for a node at a fingerprint: `<node>.<module>.<hex>`.
func Key(node, module, fingerprint string) (string, error) {
part, err := meshcatalogclient.KeyPart(fingerprint)
part, err := catalogkey.KeyPart(fingerprint)
if err != nil {
return "", err
}
@@ -234,6 +236,14 @@ func Split(node string, resources []json.RawMessage, fps map[string]string, fill
if head.Type == string(declaration.TypeFile) && head.Module != "" {
pathOf[head.Path] = key
}
if head.Module != "" {
if raw, err = withFingerprint(raw, head.Type, fps[head.Module]); err != nil {
return nil, err
}
if raw, err = withBundleFingerprints(raw, head.Type, fps); err != nil {
return nil, err
}
}
c := out[key]
c.Resources = append(c.Resources, raw)
out[key] = c
@@ -249,3 +259,93 @@ func Split(node string, resources []json.RawMessage, fps map[string]string, fill
}
return out, nil
}
// FingerprintVar is the environment value every process the node-engine starts, a daemon or a container, holds: the
// fingerprint of the want it applies. mesh-sdk's runtime reads it once at the process's start as the process's own build
// (novox/hq ADR 0656 §1, design 47 *The fingerprint handover*).
const FingerprintVar = "MESH_FINGERPRINT"
// withFingerprint is raw, when it is a daemon or a container, with FingerprintVar in its env set to fp. The value is
// the node-engine's to say, so one a module declared of that name is replaced. Another resource, or no fingerprint, is
// returned as it is.
func withFingerprint(raw json.RawMessage, kind, fp string) (json.RawMessage, error) {
if fp == "" || (kind != string(declaration.TypeProcess) && kind != string(declaration.TypeContainer)) {
return raw, nil
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(raw, &fields); err != nil {
return nil, fmt.Errorf("compositions: a %s resource is not JSON: %w", kind, err)
}
env := map[string]string{}
if had, ok := fields["env"]; ok {
if err := json.Unmarshal(had, &env); err != nil {
return nil, fmt.Errorf("compositions: the env of a %s resource is not words by name: %w", kind, err)
}
}
env[FingerprintVar] = fp
body, err := json.Marshal(env)
if err != nil {
return nil, err
}
fields["env"] = body
return json.Marshal(fields)
}
// withBundleFingerprints is raw, when it is a process whose env hands tools bundles their words (catalogue.RuntimeToolEnv),
// with each module's words holding FingerprintVar set to that module's fingerprint in fps: a bundle reads the fingerprint of
// its own module's want from the environment the tool runner starts it with (design 47 G §519). A module fps names no
// fingerprint for keeps its words, and the runner's own fingerprint reaches no bundle but its own module's (§618).
func withBundleFingerprints(raw json.RawMessage, kind string, fps map[string]string) (json.RawMessage, error) {
if kind != string(declaration.TypeProcess) {
return raw, nil
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(raw, &fields); err != nil {
return nil, fmt.Errorf("compositions: a %s resource is not JSON: %w", kind, err)
}
var env map[string]string
if had, ok := fields["env"]; !ok || json.Unmarshal(had, &env) != nil || env[catalogue.RuntimeToolModules] == "" {
return raw, nil
}
given := map[string]map[string]string{}
if had := env[catalogue.RuntimeToolEnv]; had != "" {
if err := json.Unmarshal([]byte(had), &given); err != nil {
return nil, fmt.Errorf("compositions: the %s of a process is not words by module: %w", catalogue.RuntimeToolEnv, err)
}
}
for module, words := range given {
if fp := fps[module]; fp != "" {
if words == nil {
words = map[string]string{}
given[module] = words
}
words[FingerprintVar] = fp
}
}
for module, fp := range fps { // a module with bundles and no words of its own still reads its fingerprint
if _, has := given[module]; !has && fp != "" && hasLoadedBundle(env, module) {
given[module] = map[string]string{FingerprintVar: fp}
}
}
body, err := json.Marshal(given)
if err != nil {
return nil, err
}
env[catalogue.RuntimeToolEnv] = string(body)
envBody, err := json.Marshal(env)
if err != nil {
return nil, err
}
fields["env"] = envBody
return json.Marshal(fields)
}
// hasLoadedBundle is whether the runtime's served list (catalogue.RuntimeToolModules, `module=path` entries) names a bundle of module.
func hasLoadedBundle(env map[string]string, module string) bool {
for _, entry := range strings.Split(env[catalogue.RuntimeToolModules], ",") {
if strings.HasPrefix(entry, module+"=") {
return true
}
}
return false
}
internal/judge/judge_test.go
+1 -1
View File
@@ -3,7 +3,7 @@ package judge
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"strings"
"testing"
)
internal/link/engineseat.go
+37 -15
View File
@@ -131,6 +131,28 @@ func (l *natsLink) ServeEngineSeat(handle EngineFunc) {
if handle == nil || l.conn == nil {
return
}
// Discovery names the module whose code answers: the engine's own, read from §527 and no fixed name (§634). Known at
// once from the applied record; on a first start it is served once the first composing names it.
if l.own == nil || l.own.Get() != "" {
l.serveEngineSeat(handle, l.own.Get())
return
}
ctx, cancel := context.WithCancel(context.Background())
l.engineMu.Lock()
l.stopWait = cancel
l.engineMu.Unlock()
go func() {
if name, err := l.own.Wait(ctx); err == nil {
l.serveEngineSeat(handle, name)
}
}()
}
// UseOwnName gives the link the engine's own name its discovery reads; before ServeEngineSeat.
func (l *natsLink) UseOwnName(o *OwnName) { l.own = o }
// serveEngineSeat serves the seat as the module module.
func (l *natsLink) serveEngineSeat(handle EngineFunc, module string) {
svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0",
Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true,
Metadata: map[string]string{"seat": EngineSeat, "scope": "node", "node": l.node}})
@@ -141,13 +163,15 @@ func (l *natsLink) ServeEngineSeat(handle EngineFunc) {
for _, v := range ServedEngineSeat().Serves {
verb := v.Name
answer := EngineAnswer(handle, verb, l.node, apart)
meta := EngineEndpointMetadata(v, l.node)
meta := EngineEndpointMetadata(v, l.node, module)
_ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)),
micro.WithEndpointMetadata(meta))
_ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)),
micro.WithEndpointMetadata(meta))
}
l.engineMu.Lock()
l.engine = svc
l.engineMu.Unlock()
}
// answeredApart are the verbs answered off the subscription that delivered them: each endpoint's subscription hands its
@@ -184,31 +208,32 @@ func EngineAnswer(handle EngineFunc, verb, node string, apart chan struct{}) mic
}
}
// EngineModule is the module whose code answers the seat, as discovery names it: the node-engine's manifest's.
const EngineModule = "mesh-host"
// EngineEndpointMetadata is what discovery reads of one verb's endpoint, in the shape every tool runner and the
// controller announce a seat's verb in (mesh-tools node-tools' announce package): its kind, the seat, its scope, the
// module whose code answers, the verb, the node, its description and its schema. Without the kind and the scope the
// console listed none of it, and `<node>/node-engine.<verb>` answered that nothing in the mesh is called node-engine
// (the sweep of 2026-10-30).
func EngineEndpointMetadata(v catalogue.Verb, node string) map[string]string {
func EngineEndpointMetadata(v catalogue.Verb, node, module string) map[string]string {
schema, err := json.Marshal(v.Input)
if err != nil || v.Input == nil {
schema = []byte("{}")
}
return map[string]string{"kind": "seat", "seat": EngineSeat, "scope": "node", "module": EngineModule,
return map[string]string{"kind": "seat", "seat": EngineSeat, "scope": "node", "module": module,
"tool": v.Name, "node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema)}
}
// SayEngine says one of the seat's events about this node, on the events stream, said again through a transport fault
// (sayOnce).
func (l *natsLink) SayEngine(event string, body any) error {
return l.SayEngineOn(EngineEventSubject(event, l.node), body)
}
// SayEngineOn says one of the seat's events on its subject, which names the node and, for some, the module.
func (l *natsLink) SayEngineOn(subject string, body any) error {
raw, err := json.Marshal(body)
if err != nil {
return err
}
subject := EngineEventSubject(event, l.node)
return sayOnce(func(id string) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -232,14 +257,10 @@ func sayOnce(publish func(id string) error, subject string, raw []byte) error {
// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and
// its asks' state read from the router's records under its own name.
// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the
// operator-channel's holder, the messenger (mesh-catalog modules/messenger).
const RouterRecords = "messenger_asks"
// OperatorAsks is a link that asks the operator as this node's engine. Optional.
type OperatorAsks interface {
PublishAsk(subject string, body any, id string) error
RouterRecord(key string) (*stdio.StateEntry, error)
RouterRecord(bucket, key string) (*stdio.StateEntry, error)
HearWarrants(asker string, heard func(oc.Warrant)) error
}
@@ -255,11 +276,12 @@ func (l *natsLink) PublishAsk(subject string, body any, id string) error {
return err
}
// RouterRecord reads one key of the router's records, `<asker>.<id>`; nil when it holds none.
func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) {
// RouterRecord reads one key of the router's records, `<asker>.<id>`, in the bucket the seat list names (mesh-sdk's operatorask); nil
// when it holds none.
func (l *natsLink) RouterRecord(bucket, key string) (*stdio.StateEntry, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil)
msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key, nil)
if err != nil {
return nil, err
}
internal/link/files.go
+43
View File
@@ -172,6 +172,49 @@ func (l *natsLink) GrantsObject(ctx context.Context, name string, limit int64) (
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.GrantsObject(ctx, name, limit)
}
// KeptObject reads one object of a build module's store of kept builds by its name (design 47: the build the artifact store's
// holder keeps on a build node; the build seat's verb kept-artifact names the store and the object), with the same
// subjects as FileObject, verified against its digest.
func (b OverNATS) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) {
js := b.JS
if b.Conn != nil {
wait := 10 * time.Second
if at, ok := ctx.Deadline(); ok {
if wait = time.Until(at); wait <= 0 {
return nil, context.DeadlineExceeded
}
}
bounded, err := b.Conn.JetStream(nats.MaxWait(wait))
if err != nil {
return nil, err
}
js = bounded
}
return objectByName(ctx, js, store, declaration.FileStreamPrefix+store, name, limit)
}
func (l *natsLink) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) {
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.KeptObject(ctx, store, name, limit)
}
// KeptObjects is a link that can read an object of a build module's store of kept builds.
type KeptObjects interface {
KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error)
}
// KeptObject reads an object of a build module's store of kept builds on the link open now.
func (q *Session) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
k, ok := bus.(KeptObjects)
if bus == nil || !ok {
return nil, ErrNoLink
}
return k.KeptObject(ctx, store, name, limit)
}
// GrantsObjects is a link that can read an object of the grants part's bucket of sealed user lists.
type GrantsObjects interface {
GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error)
internal/link/findings.go
+122 -9
View File
@@ -1,9 +1,13 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
"github.com/nats-io/nats.go"
)
@@ -42,19 +46,17 @@ func (q *Session) PutFinding(key string, value []byte) error {
return err
}
// HasFinding says whether one finding of this engine stands in its state now.
func (q *Session) HasFinding(key string) (bool, error) {
// FindingKeys lists the keys of this engine's findings over the link open now; none when the bucket holds none.
func (q *Session) FindingKeys() ([]string, error) {
kv, err := q.findings()
if err != nil {
return false, err
return nil, err
}
if _, err = kv.Get(key); err != nil {
if errors.Is(err, nats.ErrKeyNotFound) {
return false, nil
}
return false, err
keys, err := kv.Keys()
if err != nil && !errors.Is(err, nats.ErrNoKeysFound) {
return nil, err
}
return true, nil
return keys, nil
}
// DeleteFinding clears one finding of this engine over the link open now; a finding already gone is cleared.
@@ -76,3 +78,114 @@ func (o OverNATS) findingsKV() (nats.KeyValue, error) {
}
return kv, nil
}
// The node-engine's finding writer (mesh-sdk views.Writer, design 47 G §383, §384, §461) is made over the link open now:
// its state is the bucket above, its events are published on the engine's own stream awaiting the bus's acknowledgement, and
// its consumer of `findings-asked` is its own, from the next ask on.
// FindingsState is the node-engine's state `findings` over the link open now, as the finding writer reads it
// (views.FindingState).
type FindingsState struct{ Session *Session }
// Get is the key's entry, or none for a key the state does not hold.
func (s FindingsState) Get(key string) (*stdio.StateEntry, error) {
kv, err := s.Session.findings()
if err != nil {
return nil, err
}
e, err := kv.Get(key)
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
}
// Put writes the value as JSON under the key.
func (s FindingsState) Put(key string, value any) (uint64, error) {
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
kv, err := s.Session.findings()
if err != nil {
return 0, err
}
return kv.Put(key, raw)
}
// Delete deletes the key; a key already gone is deleted.
func (s FindingsState) Delete(key string) error { return s.Session.DeleteFinding(key) }
// Keys lists the keys the state holds.
func (s FindingsState) Keys() ([]string, error) { return s.Session.FindingKeys() }
var _ views.FindingState = FindingsState{}
// eventsOver is a link that publishes on a stream and makes a consumer of its own on another.
type eventsOver interface {
publishEvent(ctx context.Context, subject string, data []byte, id string) error
listenFrom(stream, subject string, handle func([]byte)) (func(), error)
}
func publishEvent(js nats.JetStreamContext, ctx context.Context, subject string, data []byte, id string) error {
_, err := js.Publish(subject, data, nats.MsgId(id), nats.Context(ctx))
return err
}
func listenFrom(js nats.JetStreamContext, stream, subject string, handle func([]byte)) (func(), error) {
sub, err := js.Subscribe(subject, func(m *nats.Msg) { handle(m.Data) }, nats.BindStream(stream), nats.DeliverNew(), nats.AckNone())
if err != nil {
return nil, fmt.Errorf("a consumer of %s: %w", subject, err)
}
return func() { _ = sub.Unsubscribe() }, nil
}
func (l *natsLink) publishEvent(ctx context.Context, subject string, data []byte, id string) error {
return publishEvent(l.js, ctx, subject, data, id)
}
func (l *natsLink) listenFrom(stream, subject string, handle func([]byte)) (func(), error) {
return listenFrom(l.js, stream, subject, handle)
}
func (o OverNATS) publishEvent(ctx context.Context, subject string, data []byte, id string) error {
return publishEvent(o.JS, ctx, subject, data, id)
}
func (o OverNATS) listenFrom(stream, subject string, handle func([]byte)) (func(), error) {
return listenFrom(o.JS, stream, subject, handle)
}
func (q *Session) events() (eventsOver, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
e, ok := bus.(eventsOver)
if bus == nil || !ok {
return nil, ErrNoLink
}
return e, nil
}
// PublishEvent publishes one message awaiting the bus's acknowledgement, over the link open now (views.WriterConfig.Publish).
func (q *Session) PublishEvent(ctx context.Context, subject string, data []byte, id string) error {
e, err := q.events()
if err != nil {
return err
}
return e.publishEvent(ctx, subject, data, id)
}
// ListenFrom makes a consumer of its own on stream for subject that delivers only the messages published after it is
// made, over the link open now, and answers how to end it (views.WriterConfig.Listen).
func (q *Session) ListenFrom(stream, subject string, handle func([]byte)) (func(), error) {
e, err := q.events()
if err != nil {
return nil, err
}
return e.listenFrom(stream, subject, handle)
}
internal/link/handacts.go
+69 -16
View File
@@ -1,6 +1,7 @@
package link
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
@@ -55,18 +56,17 @@ const (
// (novox/hq ADR 0636 §7). The node-engine seat's definition carries both verbs.
ActSeatList = nodeengine.VerbSeatList
ActSeatListAgain = nodeengine.VerbSeatListAgain
// ActFirstViews reads the node-engine's first views: the views it follows, the entries it keeps of the builds its wants
// and runnings name, and the wants it composes from; ActFirstViewAgain asks a view again and ActEntryAgain an entry
// (design 47 G §422, §698).
ActFirstViews = nodeengine.VerbFirstViews
ActFirstViewAgain = nodeengine.VerbFirstViewAgain
ActEntryAgain = nodeengine.VerbEntryAgain
// ActProcessBuilds reads the build each process on this node was started with (design 47, *The fingerprint handover*);
// it reads only; who may call it is the seat definition's (design 47 G §440, §447).
ActProcessBuilds = nodeengine.VerbProcessBuilds
)
// SeatListCallers are who may call the engine's two seat-list verbs (novox/hq ADR 0636 §7; the node-engine seat's
// definition: the operator and the healers): every operator's session and the healers, both verbs alike. A call through
// the mesh MCP server, and nox, is proved only as its machine's tool runner (<node>.node-tools, issue 627), so node-tools
// stands for an operator's session until a session is proved; without it the verbs answered nobody but the healers, and
// no healer calls them (issue 1966).
var SeatListCallers = stdio.VerbCallers{
ActSeatList: {stdio.AnyPerson, "node-tools", "mesh-healers"},
ActSeatListAgain: {stdio.AnyPerson, "node-tools", "mesh-healers"},
}
// The states of an act: asked, done and refused are also the seat's events.
const (
HandAsked = nodeengine.EventAsked
@@ -166,6 +166,12 @@ type HandActs struct {
// SeatList answers the seat list the engine holds; SeatListAgain makes it read the list now and answers what it holds.
SeatList func() (any, error)
SeatListAgain func() (any, error)
// FirstViews answers the verb first-views; EntryAgain asks one entry of the catalog half again, refusing in words one the
// engine keeps none of.
FirstViews func() (any, error)
EntryAgain func(ctx context.Context, fingerprint string) error
// ProcessBuilds answers the build each process of the node was started with, narrowed to a module when one is given.
ProcessBuilds func(module string) (any, error)
// ComposeCandidate composes a check's candidate (the indexer's candidate key) on this node in shadow and answers per
// module, applying and writing nothing; with input, also the composition input it composed from (the verb
// compose-candidate, ADR 0593 §7).
@@ -237,17 +243,31 @@ func (h *HandActs) Call(c EngineCall) (any, error) {
}
return h.Compose(arg("module"))
case ActSeatList:
if err := SeatListCallers.Allowed(c.Verb, c.Caller); err != nil {
return nil, err
}
if h.SeatList == nil {
return nil, errors.New("this engine reads no seat list yet: its node agent has not started")
}
return h.SeatList()
case ActSeatListAgain:
if err := SeatListCallers.Allowed(c.Verb, c.Caller); err != nil {
return nil, err
case ActFirstViews:
if h.FirstViews == nil {
return nil, errors.New("this engine keeps no first views yet: its node agent has not started")
}
return h.FirstViews()
case ActFirstViewAgain:
if arg("owner") == "" || arg("state") == "" || arg("why") == "" {
return nil, errors.New("first-view-again names the owner, the state and why")
}
// The node-engine follows no branch and no view of another part's state: the views it holds are its entries (entry-again)
// and its seat list (seat-list-again).
return nil, fmt.Errorf("this engine follows no first view of %s's %s: its entries are asked again by entry-again, and its seat list by seat-list-again",
arg("owner"), arg("state"))
case ActEntryAgain:
return h.AskEntryAgain(c.Caller, arg("fingerprint"), arg("why"))
case ActProcessBuilds:
if h.ProcessBuilds == nil {
return nil, errors.New("this engine reads no process yet: its node agent has not started")
}
return h.ProcessBuilds(arg("module"))
case ActSeatListAgain:
if h.SeatListAgain == nil {
return nil, errors.New("this engine reads no seat list yet: its node agent has not started")
}
@@ -290,6 +310,39 @@ func (h *HandActs) Call(c EngineCall) (any, error) {
return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb)
}
// AskEntryAgain asks fingerprint's entry of the catalog half again by hand: a hand act recorded with who asked and why, then
// done. The answer is the ask and the verb that reads where it stands (design 47 G §698).
func (h *HandActs) AskEntryAgain(caller, fingerprint, why string) (any, error) {
if h.EntryAgain == nil {
return nil, errors.New("this engine reads no entry by fingerprint yet: its node agent has not started")
}
if fingerprint == "" || why == "" {
return nil, errors.New("entry-again names the build's fingerprint and why")
}
ctx, cancel := context.WithTimeout(context.Background(), VaultWithin)
defer cancel()
a := HandAct{ID: h.id(ActEntryAgain), Act: ActEntryAgain, Subject: fingerprint, Caller: caller, Why: why, At: h.now().UTC(), State: HandDone}
err := h.EntryAgain(ctx, fingerprint)
if err != nil {
a.State, a.Said = HandRefused, err.Error()
}
// A done ask is kept as an act; a refused one, which any caller may cause, rolls with the judgements.
record := h.record
if err != nil {
record = h.roll
}
if rerr := record(a); rerr != nil {
return nil, rerr
}
if err != nil {
return nil, err
}
return map[string]any{
"asked": map[string]any{"v": sdkkeys.Version, "fingerprint": fingerprint, "why": why, "by": caller, "at": a.At},
"read": ActFirstViews,
}, nil
}
// ---- the record --------------------------------------------------------------------------------------------------
// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key,
internal/link/handover_test.go
+33 -8
View File
@@ -271,6 +271,20 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
ManifestReads: func() ([]string, time.Time) {
return []string{"form:a", "seat:b"}, time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC)
},
// Design 47 G §422, §698: the first views and the entries asked again.
FirstViews: func() (any, error) {
return map[string]any{"first-views": []any{}, "entries": []any{}, "wants": []any{}}, nil
},
EntryAgain: func(_ context.Context, fingerprint string) error {
if fingerprint == "src2:gone" {
return errors.New("no entry of src2:gone is kept")
}
return nil
},
// Design 47: the build each process was started with, narrowed by module.
ProcessBuilds: func(module string) (any, error) {
return map[string]any{"processes": []any{}, "module": module}, nil
},
// ADR 0565 §6, §15: the node's key of the engine's state packages.
Packages: func(write bool) (any, error) {
key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`)
@@ -301,16 +315,23 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`},
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`},
{"manifest-reads", EngineToolSubject("manifest-reads", "n1"), `"manifest-reads":["form:a","seat:b"],"node":"n1","stated-at":"2026-11-04T10:00:00Z","v":1`},
// ADR 0636 §7: both seat-list verbs answer every operator's session, proved as a person or, through the mesh MCP
// server, as its machine's tool runner (issue 627, issue 1966), and the healers, no other module.
// Design 47 G §447: the engine keeps no callers table; who may call a verb is the seat definition's, which the
// grants part makes the bus enforce, so a call that reaches the engine is answered whoever the caller is.
{"seat-list", "mesh.seat.node-engine.call.seat-list.n1.person~jochen", `"held":true`},
{"seat-list", "mesh.seat.node-engine.call.seat-list.n1.shanks~node-tools", `"held":true`},
{"seat-list", "mesh.seat.node-engine.call.seat-list.n1.ace~messenger", `not on the holder's caller list for seat-list`},
{"seat-list", "mesh.seat.node-engine.call.seat-list.n1.ace~messenger", `"held":true`},
{"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.shanks~node-tools", `"read":true`},
{"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.person~jochen", `"read":true`},
{"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~mesh-healers", `"read":true`},
{"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~messenger", `not on the holder's caller list for seat-list-again`},
{"seat-list-again", EngineToolSubject("seat-list-again", "n1"), `no caller the tool runner proved`},
{"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~messenger", `"read":true`},
{"first-views", "mesh.seat.node-engine.call.first-views.n1.person~jochen", `"entries":[],"first-views":[],"wants":[]`},
{"first-view-again", "mesh.seat.node-engine.call.first-view-again.n1.person~jochen " + `{"owner":"mesh-catalog","state":"modules","why":"doubt"}`, `follows no first view of mesh-catalog's modules`},
{"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:a","why":"doubt"}`, `"fingerprint":"src2:a","v":1,"why":"doubt"},"read":"first-views"`},
{"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:gone","why":"doubt"}`, `no entry of src2:gone is kept`},
{"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:a"}`, `entry-again names the build's fingerprint and why`},
{"process-builds", "mesh.seat.node-engine.call.process-builds.n1.person~jochen " + `{"module":"web"}`, `"module":"web","processes":[]`},
{"process-builds", "mesh.seat.node-engine.call.process-builds.n1.ace~messenger", `"processes":[]`},
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
} {
subject, body, ok := strings.Cut(row.subject, " ")
@@ -323,8 +344,8 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
}
}
acts, _ := h.Acts(0)
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" ||
got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories {
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 5 || got[2].Caller != "unproven" ||
got[4].Caller != "shanks.node-tools" || got[4].Act != ActJudgeDirectories || got[0].Act != ActEntryAgain || got[0].State != HandRefused {
t.Errorf("the judgements are recorded as %+v", got)
}
})
@@ -462,6 +483,10 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) {
})
// 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery.
t.Run("the node seat is served on this node and discovered", func(t *testing.T) {
// Discovery names the engine's own module as §527 reads it, no fixed name (design 47 G §634, A-149).
own := NewOwnName()
own.Set("the-engine")
l.UseOwnName(own)
l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil })
reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) {
@@ -488,7 +513,7 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) {
announced := 0
for _, e := range said.Endpoints {
md := e.Metadata
if md["kind"] != "seat" || md["scope"] != "node" || md["seat"] != EngineSeat || md["module"] != "mesh-host" ||
if md["kind"] != "seat" || md["scope"] != "node" || md["seat"] != EngineSeat || md["module"] != "the-engine" ||
md["node"] != m.Node || md["description"] == "" || !strings.HasPrefix(md["schema"], "{") {
t.Errorf("the endpoint on %s announces %v", e.Subject, md)
}
internal/link/have.go
+147 -3
View File
@@ -67,6 +67,11 @@ type haveState struct {
userList map[string]nodeengine.UserListWritten
// userListNoKey is each module whose user list was said to have no have key to ride on, so it is said once.
userListNoKey map[string]bool
// runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once.
runningSaid map[string]string
// waitSaid is, by module, the wait its event wait-held last said (since, step, reason), so each is said once and its end
// is said when the have no longer has it (design 47 G §541, §629, A-113).
waitSaid map[string]nodeengine.Wait
// unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look
// has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before.
unjudged map[string]bool
@@ -306,7 +311,7 @@ func (q *Session) nodeFactsWithRecord(r Report, before *nodeengine.HaveByNodeNod
n := nodeFacts(r)
if q.Keys != nil {
keys := q.Keys()
keys.Stated = q.runnerStated
keys.Stated, keys.StatedBy = q.runnerStated, q.runnerFingerprint
n.Record = recordOf(q.Membership.Node, r, keys)
}
if before != nil && before.Record != nil {
@@ -360,6 +365,14 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
return
}
s := &q.haves
// What the pass says as the seat's events (running) is said once the haves' lock is let go: a
// publish never holds it.
var said []func()
defer func() {
for _, say := range said {
say()
}
}()
s.mu.Lock()
defer s.mu.Unlock()
if s.modules == nil && s.node == nil && len(s.removed) == 0 && len(s.userList) == 0 {
@@ -399,6 +412,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
if h.Phase == nodeengineclient.PhaseFailed {
h.PullWait = f.PullWait
}
h.Wait = f.Wait
h.Health = s.healthOf(module, f)
if h.Detail == "" {
h.Detail = s.wordsOf(module, h.Health)
@@ -426,8 +440,21 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
s.written = map[string]string{}
}
s.written[module] = f.Build
if say, _ := q.haveSaid(); say != nil {
said = append(said, func() { say(module, h) })
}
if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok {
said = append(said, func() { q.sayRunning(ev) })
}
if held, ended, ok := s.waitToSay(module, h.Wait); ok {
if h.Wait != nil {
said = append(said, func() { q.sayWait(nodeengine.EventWaitHeld, nodeengine.SubjectWaitHeld(node, module), held) })
} else {
said = append(said, func() { q.sayWait(nodeengine.EventWaitEnded, nodeengine.SubjectWaitEnded(node, module), ended) })
}
}
}
if !q.writeRemoved(write, bucket, node) {
if !q.writeRemoved(write, bucket, node, &said) {
return
}
if !q.writeUserListsAlone(write, bucket, node) {
@@ -501,7 +528,7 @@ func (q *Session) haveSucceeded() {
// writeRemoved writes, once, each module the applied declaration no longer holds and this node-engine wrote a have key
// for, with the phase removed over the revision it reads (issue 1369). The key is never deleted, and a key that is not
// in the bucket is not created. False when the pass should stop. The caller holds the lock of the haves.
func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) bool {
func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, said *[]func()) bool {
s := &q.haves
gone := make([]string, 0, len(s.removed))
for module := range s.removed {
@@ -531,6 +558,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
switch {
case errors.Is(err, ErrHaveAbsent):
delete(s.removed, module) // nothing to say: the key is not there
if _, gone := q.haveSaid(); gone != nil {
*said = append(*said, func() { gone(module) })
}
continue
case err != nil:
q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err))
@@ -539,6 +569,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
}
if c, err := nodeengineclient.DecodeHave(current); err == nil && c.Phase == nodeengineclient.PhaseRemoved {
delete(s.removed, module)
if _, gone := q.haveSaid(); gone != nil {
*said = append(*said, func() { gone(module) })
}
continue
}
if err := b.Set(ctx, key, raw, revision); err != nil {
@@ -547,10 +580,22 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
return false
}
delete(s.removed, module)
if _, gone := q.haveSaid(); gone != nil {
*said = append(*said, func() { gone(module) })
}
// The module no longer runs here: running says stopped, with no fingerprint, on its own subject (design 47 G §365, §404).
delete(s.runningSaid, module)
*said = append(*said, func() { q.sayRunning(stoppedRunning(node, module)) })
}
return true
}
// stoppedRunning is the event running of a module the node-engine's have no longer holds: stopped, with no fingerprint
// and no manifest (design 47 G §365, §404).
func stoppedRunning(node, module string) nodeengine.Running {
return nodeengine.Running{V: sdkkeys.Version, Node: node, Module: module, Stopped: true}
}
// writeUserListsAlone writes the user list of each module this node-engine holds no fact of (no apply here said its have)
// into that module's key as the bucket holds it, keeping every other field the key says (novox/hq ADR 0547 §8). The have
// of a module is one value with parts from more than one source: the apply's (fingerprint, digest, build, phase, health)
@@ -885,3 +930,102 @@ func nothingJudged(f ModuleFact, looked bool) string {
}
return nodeengineclient.HealthUnknown
}
// runningToSay is the event running for a module whose have was just written running with a health of ok or failed at a
// fingerprint whose manifest is known, when it has not been said for that fingerprint, manifest and health (design 47 G
// §520): `healthy` says the build's health, and it is said again each time that changes and, the said set being empty, for
// every module the have holds at the node-engine's start. A module whose health is waits or unknown, or whose manifest the
// node does not know, says nothing, and is said at the write that knows. Under s.mu.
func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleValue, manifest func(module, fp string) string) (nodeengine.Running, bool) {
if h.Phase != nodeengineclient.PhaseRunning || h.Fp == "" || manifest == nil {
return nodeengine.Running{}, false
}
if h.Health != nodeengineclient.HealthOK && h.Health != nodeengineclient.HealthFailed {
return nodeengine.Running{}, false
}
digest := manifest(module, h.Fp)
if digest == "" {
return nodeengine.Running{}, false
}
healthy := h.Health == nodeengineclient.HealthOK
said := fmt.Sprintf("%s\x00%s\x00%t", h.Fp, digest, healthy)
if s.runningSaid[module] == said {
return nodeengine.Running{}, false
}
if s.runningSaid == nil {
s.runningSaid = map[string]string{}
}
s.runningSaid[module] = said
return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest, Healthy: &healthy}, true
}
// waitToSay is the event wait-held for a have written with a wait the last one did not say, or wait-ended for one written with
// none after a wait said (design 47 G §541, §629): a wait of the same step, reason and since is said once. Under s.mu.
func (s *haveState) waitToSay(module string, w *nodeengine.Wait) (held, ended nodeengine.Wait, ok bool) {
last, said := s.waitSaid[module]
if w == nil {
if !said {
return held, ended, false
}
delete(s.waitSaid, module)
return held, last, true
}
if said && last.Step == w.Step && last.Reason == w.Reason && last.Since.Equal(w.Since) && last.Words == w.Words {
return held, ended, false
}
if s.waitSaid == nil {
s.waitSaid = map[string]nodeengine.Wait{}
}
s.waitSaid[module] = *w
return *w, ended, true
}
// sayWait says the event wait-held or wait-ended of a wait, with the time its bound counts from.
func (q *Session) sayWait(event, subject string, w nodeengine.Wait) {
if w.ClockFrom == nil {
since := w.Since
w.ClockFrom = &since
}
q.sayEngineOn(subject, event, w)
}
// sayRunning says the event running, on its subject `running.<node>.<module>`.
func (q *Session) sayRunning(ev nodeengine.Running) {
ev.Node, ev.At = q.Membership.Node, time.Now().UTC()
q.sayEngineOn(nodeengine.SubjectRunning(ev.Node, ev.Module), "running", ev)
}
// SetHaveSaid gives the session what it tells of each module's have as it writes it (said), and of a module whose have says
// it is gone (gone): the node-engine's findings of itself are made of them (design 47 G A-197). Either may be nil.
func (q *Session) SetHaveSaid(said func(module string, h nodeengine.HaveByNodeModuleValue), gone func(module string)) {
q.haveSay.Lock()
q.haveSaidFn, q.haveGoneFn = said, gone
q.haveSay.Unlock()
}
func (q *Session) haveSaid() (func(string, nodeengine.HaveByNodeModuleValue), func(string)) {
q.haveSay.Lock()
defer q.haveSay.Unlock()
return q.haveSaidFn, q.haveGoneFn
}
// SetManifestOf gives the session what names the manifest of the build a module runs (the composer's picture of the
// catalog half): its digest, or "" when the node does not know it.
func (q *Session) SetManifestOf(f func(module, fp string) string) {
q.manifest.Lock()
q.manifestFn = f
q.manifest.Unlock()
}
func (q *Session) manifestOf() func(module, fp string) string {
q.manifest.Lock()
defer q.manifest.Unlock()
return q.manifestFn
}
// SayComposed says the event composed for a composition written at a fingerprint, on `composed.<node>`.
func (q *Session) SayComposed(module, fingerprint, digest string) {
node := q.Membership.Node
q.sayEngineOn(nodeengine.SubjectComposed(node), "composed", nodeengine.Composed{V: sdkkeys.Version, Node: node,
Module: module, Fingerprint: fingerprint, Digest: digest, At: time.Now().UTC()})
}
internal/link/have_test.go
+145 -5
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
sdkkeys "git.novox.be/novox/mesh-sdk/go/keys"
"github.com/novox/mesh-host/internal/compositions"
"slices"
@@ -43,14 +45,45 @@ type haveLink struct {
readsSaid []nodeengine.ManifestReadsChangedEvent
// sayFails fails that many of the node seat's says, as a publish past the bus's retry bound; said is every event said.
sayFails int
// sayFault is what a failed say answers; a bus answering a refusal when nil.
sayFault error
said []string
// onSaid are the events said on a subject of their own (running), by subject, in order.
onSaid []string
// asked are the subjects asked of the bus, records the router's records read as bucket and key.
asked, records []string
}
func (l *haveLink) SayEngineOn(subject string, body any) error {
raw, _ := json.Marshal(body)
l.hmu.Lock()
l.onSaid = append(l.onSaid, subject+" "+string(raw))
l.hmu.Unlock()
return nil
}
// saidOn is the events said on a subject, as the bus would carry them.
func (l *haveLink) saidOn(prefix string) []string {
l.hmu.Lock()
defer l.hmu.Unlock()
var out []string
for _, e := range l.onSaid {
if strings.HasPrefix(e, prefix) {
out = append(out, e)
}
}
return out
}
func (l *haveLink) SayEngine(event string, body any) error {
l.hmu.Lock()
if l.sayFails > 0 {
l.sayFails--
fault := l.sayFault
l.hmu.Unlock()
if fault != nil {
return fault
}
return errors.New("nats: timeout")
}
l.said = append(l.said, event)
@@ -74,6 +107,23 @@ type haveEntry struct {
rev uint64
}
// The link as the operator's asker and a tool bus: the seat list answers the holder of the seat operator-channel, and the
// router's records are read from the bucket named for it (design 47 G A-101).
func (l *haveLink) Ask(_ context.Context, subject string, _ []byte) ([]byte, error) {
l.hmu.Lock()
l.asked = append(l.asked, subject)
l.hmu.Unlock()
return []byte(`{"result":{"seats":[{"seat":"operator-channel","holders":[{"module":"the-router"}]}]}}`), nil
}
func (l *haveLink) PublishAsk(string, any, string) error { return nil }
func (l *haveLink) HearWarrants(string, func(oc.Warrant)) error { return nil }
func (l *haveLink) RouterRecord(bucket, key string) (*stdio.StateEntry, error) {
l.hmu.Lock()
l.records = append(l.records, bucket+" "+key)
l.hmu.Unlock()
return nil, nil
}
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
@@ -186,7 +236,13 @@ func aHaveQueue(l *haveLink) (*Session, *[]string) {
func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
// Design 47 G A-197: each have as written is told to the node-engine's findings of itself, the failed one with its detail.
told := map[string]nodeengine.HaveByNodeModuleValue{}
q.SetHaveSaid(func(m string, h nodeengine.HaveByNodeModuleValue) { told[m] = h }, nil)
q.SayFacts(context.Background(), appliedModules())
if told["b"].Phase != nodeengineclient.PhaseFailed || !strings.Contains(told["b"].Detail, "no space") || told["a"].Phase != nodeengineclient.PhaseRunning {
t.Errorf("the haves told were %+v", told)
}
a, b := l.have(t, "n1.a"), l.have(t, "n1.b")
if a.Phase != nodeengineclient.PhaseRunning || a.Build != "b-1" || a.Health != nodeengineclient.HealthOK {
t.Errorf("a: %+v", a)
@@ -270,7 +326,7 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile.
func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
q, journal := aHaveQueue(l)
rep := appliedModules()
rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}}
rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}}
@@ -300,7 +356,8 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
q.Keys = func() RecordKeys { return keys }
l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls"} // the tool runner's calls (ENG-2)
l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls", // the tool runner's calls (ENG-2)
FingerprintKey: "sha256:runner"} // and the runner that stated them (design 46 approval mark §6)
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
"machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}}
@@ -311,9 +368,10 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
t.Fatalf("the node key holds no record: %+v, %v", got, err)
}
wantRecord := nodeengine.NodeRecord{V: sdkkeys.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd", CapabilityWantParts}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Networks: []nodeengine.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract,
ManifestReads: catalogue.ManifestReads()} // what it reads of a manifest: its contract's (ADR 0614 §4)
ManifestReads: catalogue.ManifestReads(), // what it reads of a manifest: its contract's (ADR 0614 §4)
RunnerCapabilities: []string{"bus-state-calls", "ledger-calls"}, RunnerFingerprint: "sha256:runner"}
if !nodeengineclient.SameNodeRecord(*got.Record, wantRecord) {
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
}
@@ -330,6 +388,10 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
if l.kv["n1.node"].rev != at {
t.Errorf("a report changing nothing wrote the node key again")
}
// Design 47 G §590: the node's capabilities are said by the event manifest-reads-changed alone; no capabilities-changed.
if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed."); len(got) != 0 {
t.Errorf("capabilities-changed said %v, want none (G §590)", got)
}
// A runner that does not answer states what it stated before; one answering without any states none (an older runner).
l.statedErr = fmt.Errorf("%w: nothing answered", ErrNoAnswer)
l.stated = nil
@@ -339,7 +401,7 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
}
l.statedErr, l.stated = nil, map[string]string{}
q.SayFacts(context.Background(), rep)
if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd"}) {
if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd", "want-parts"}) {
t.Errorf("a runner stating no calls: revision %d (was %d), capabilities %v", l.kv["n1.node"].rev, at, got.Record.Capabilities)
}
at++
@@ -413,6 +475,41 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
catalogue.UseSeats(all)
q.RestateReads(context.Background())
// A-292 (§590, issue 1975): a restatement the bus did not take, past its retry bound, is said again on the same link
// until the bus takes it, never held for the next link, and each failed attempt is in the engine's journal by name.
saidBefore = len(l.readsSaid)
l.hmu.Lock()
l.sayFails, l.sayFault = 2, fmt.Errorf("saying: %w", nats.ErrTimeout)
l.hmu.Unlock()
catalogue.UseSeats(slices.DeleteFunc(slices.Clone(all), func(s catalogue.Seat) bool { return s.Name == "mesh-vault" }))
q.RestateReads(context.Background())
q.retrying.Wait()
l.hmu.Lock()
again := slices.Clone(l.readsSaid[saidBefore:])
l.sayFault = nil
l.hmu.Unlock()
if len(again) != 1 || slices.Contains(again[0].ManifestReads, "seat:mesh-vault") {
t.Errorf("a restatement the bus did not take, said on this link: %+v, want it said once with the new statement", again)
}
failed := 0
for _, line := range *journal {
if strings.Contains(line, "could not say the node seat's "+nodeengine.EventManifestReadsChanged) &&
strings.Contains(line, "on this link") {
failed++
}
}
if failed != 2 {
t.Errorf("the journal names %d failed attempts of %s, want 2: %q", failed, nodeengine.EventManifestReadsChanged, *journal)
}
q.reads.Lock()
_, held := q.reads.unsaid[nodeengine.EventManifestReadsChanged]
q.reads.Unlock()
if held {
t.Errorf("a restatement said on this link is also held for the next link")
}
catalogue.UseSeats(all)
q.RestateReads(context.Background())
// And the publish itself (sayOnce): through no responders, as the old JetStream client says it, and a deadline it is published again under one message id,
// so the stream takes it once.
t.Cleanup(busretry.Waits(10*time.Second, time.Millisecond, 2*time.Millisecond))
@@ -519,9 +616,14 @@ func onlyA() Report {
func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
var gone []string
q.SetHaveSaid(nil, func(m string) { gone = append(gone, m) })
q.SayFacts(context.Background(), appliedModules())
revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev
q.SayFacts(context.Background(), onlyA())
if len(gone) != 1 || gone[0] != "b" { // design 47 G A-197: a module that left ends its findings
t.Errorf("the modules told gone were %v, want b", gone)
}
b := l.have(t, "n1.b")
if b.Phase != nodeengineclient.PhaseRemoved || b.Build != "b-2" {
t.Errorf("b: %+v", b)
@@ -529,6 +631,10 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) {
if l.kv["n1.b"].rev != revB+1 {
t.Errorf("b was written at revision %d, want one over %d", l.kv["n1.b"].rev, revB)
}
// Design 47 G §365: a module the have no longer holds says running stopped, with no fingerprint, once, on its own subject.
if got := l.saidOn("mesh.seat.node-engine.event.running.n1.b "); len(got) != 1 || !strings.Contains(got[0], `"stopped":true`) || strings.Contains(got[0], `"fingerprint"`) {
t.Errorf("running said %v for b, want one stopped with no fingerprint", got)
}
if l.kv["n1.a"].rev != revA || l.have(t, "n1.a").Phase != nodeengineclient.PhaseRunning {
t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev)
}
@@ -663,6 +769,36 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
if h.Fp != fp || h.Composed != dig || h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthUnknown {
t.Fatalf("the have after the apply: %+v", h)
}
// Design 47, *What each step says*: running is said once, when the build is healthy and its manifest is known, on
// running.<node>.<module>, with the module, the fingerprint and the manifest's digest; never while it is unknown.
q.SetManifestOf(func(module, f string) string {
if module == "web" && f == fp {
return "sha256:" + strings.Repeat("c", 64)
}
return ""
})
q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy}))
q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy}))
if got := l.saidOn("mesh.seat.node-engine.event.running.n1.web "); len(got) != 1 ||
!strings.Contains(got[0], `"healthy":true`) || !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) {
t.Fatalf("running said %v, want one for the healthy build", got)
}
// A have with a wait says wait-held once on wait-held.<node>.<module>, and wait-ended when the wait is gone (design 47 G §541, A-113).
hold := &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "restore point missing", Since: time.Now().UTC(), Words: "w"}
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold})
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold})
if got := l.saidOn("mesh.seat.node-engine.event.wait-held.n1.web "); len(got) != 1 || !strings.Contains(got[0], `"reason":"restore point missing"`) || !strings.Contains(got[0], `"clock-from"`) {
t.Fatalf("wait-held said %v, want one", got)
}
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig})
if got := l.saidOn("mesh.seat.node-engine.event.wait-ended.n1.web "); len(got) != 1 {
t.Fatalf("wait-ended said %v, want one", got)
}
// The router's record of an ask is read from the bucket of the seat's holder on the seat list (design 47 G A-101).
if _, err := q.RouterRecord("engine.a1"); err != nil || !slices.Equal(l.records, []string{"the-router_asks engine.a1"}) ||
!strings.Contains(strings.Join(l.asked, " "), "assignments") {
t.Fatalf("the router's record read as %v after asking %v: %v", l.records, l.asked, err)
}
// Starting: it waits. Past its start window the check fails: failed.
q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting}))
if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig {
@@ -672,6 +808,10 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthFailed || got.Fp != fp || got.Composed != dig {
t.Fatalf("a failing check after the start window: %+v", got)
}
// §520: running is said again when healthy changes, now false, and the stream's newest says it.
if got := l.saidOn("mesh.seat.node-engine.event.running.n1.web "); len(got) != 2 || !strings.Contains(got[1], `"healthy":false`) {
t.Fatalf("running said %v after the check failed, want a second with healthy false", got)
}
// The bus's user list written alone rides on the module's have, and an apply after it keeps it (ADR 0547 §8).
w := nodeengine.UserListWritten{Source: "n1.web." + fp, Revision: 4, Wanted: dig, Digest: dig, Users: 3,
internal/link/hearing_nats.go
+10
View File
@@ -8,6 +8,7 @@ import (
"os"
"regexp"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
@@ -48,6 +49,10 @@ type natsLink struct {
moving *nats.Subscription
// engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563).
engine micro.Service
engineMu sync.Mutex
// own is the engine's own module, which discovery names (§634); stopWait ends a wait for it when the link closes.
own *OwnName
stopWait context.CancelFunc
warrants *nats.Subscription
node string
// user is the bus user this link authenticated as (node.<n> or node.<n>--b, ADR 0583 §1).
@@ -295,9 +300,14 @@ func (l *natsLink) Close() {
if l.judging != nil {
_ = l.judging.Unsubscribe()
}
l.engineMu.Lock()
if l.stopWait != nil {
l.stopWait()
}
if l.engine != nil {
_ = l.engine.Stop()
}
l.engineMu.Unlock()
if l.warrants != nil {
_ = l.warrants.Unsubscribe()
}
internal/link/messages.go
+2
View File
@@ -647,4 +647,6 @@ type ModuleFact struct {
Sealed compositions.Seals
// PullWait is, on a failed fact, the pull from a store its apply waits on (nodeengine.HaveByNodeModuleValue.PullWait, issue 1966 ENG-9).
PullWait *nodeengine.PullWait
// Wait is the hold or wait the module has, with the ask-at and act of a recovery wait (nodeengine.HaveByNodeModuleValue.Wait, design 47 G §560).
Wait *nodeengine.Wait
}
internal/link/movesecret_test.go
+25
View File
@@ -139,4 +139,29 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) {
if _, err := (&Session{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) {
t.Fatalf("a queue with no link asked the vault: %v", err)
}
// AM-H1 (design 46 approval mark §6, lead answer A-239): the composition reads what the grants part withholds through
// its verb withheld, asked in the engine's name, and says withheld unknown when nothing answers.
grants, err := conn.Subscribe("mesh.seat.mesh-grants.call.withheld.node~"+m.Node, func(msg *nats.Msg) {
_ = msg.Respond([]byte(`{"result":{"node":"` + m.Node + `","modules":[{"module":"shell","subjects":[` +
`{"subject":"mesh.mod.shell.tool.execute","why":"its tool runner does not state approval-mark/1"}]}]}}`))
})
if err != nil {
t.Fatal(err)
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
if got, unknown := q.WithheldOf(t.Context(), "shell"); unknown != "" || len(got) != 1 || got[0].Subject != "mesh.mod.shell.tool.execute" {
t.Errorf("withheld of shell = %+v (%q)", got, unknown)
}
if got, unknown := q.WithheldOf(t.Context(), "other"); unknown != "" || len(got) != 0 {
t.Errorf("withheld of a module the grants part names none of = %+v (%q)", got, unknown)
}
_ = grants.Unsubscribe()
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
if got, unknown := q.WithheldOf(t.Context(), "shell"); unknown != WithheldUnknown || got != nil {
t.Errorf("with the grants part down, withheld = %+v (%q)", got, unknown)
}
}
internal/link/ownname.go
+61
View File
@@ -0,0 +1,61 @@
package link
import (
"context"
"errors"
"sync"
)
// OwnName is the module that is this node-engine: the module whose catalog entry claims the seat node-engine, which the
// engine learns first from its applied record of the build whose fingerprint is its own and, on a first start with no such
// record, from its first composing (design 47 G §527, §634, A-149). Its discovery, its finding writer and its login names
// in the vault read it here and fix none.
type OwnName struct {
mu sync.Mutex
name string
ready chan struct{}
}
// NewOwnName makes an own name not yet known.
func NewOwnName() *OwnName { return &OwnName{ready: make(chan struct{})} }
// Set keeps the name; an empty one changes nothing. The first name opens Wait for everyone.
func (o *OwnName) Set(name string) {
if o == nil || name == "" {
return
}
o.mu.Lock()
defer o.mu.Unlock()
if o.name == "" {
close(o.ready)
}
o.name = name
}
// Get is the name, "" while it is not known.
func (o *OwnName) Get() string {
if o == nil {
return ""
}
o.mu.Lock()
defer o.mu.Unlock()
return o.name
}
// Wait returns the name once it is known, or ctx's error.
func (o *OwnName) Wait(ctx context.Context) (string, error) {
if o == nil {
return "", errors.New("this session names no own module")
}
select {
case <-o.ready:
return o.Get(), nil
case <-ctx.Done():
return "", ctx.Err()
}
}
// OwnNamed is a link that reads the node-engine's own module from an OwnName when it serves its seat.
type OwnNamed interface {
UseOwnName(*OwnName)
}
internal/link/ownname_test.go
+32
View File
@@ -0,0 +1,32 @@
package link
import (
"context"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Design 47 G §527, §634: discovery names the module the engine reads of itself, Wait holds until it is known, and an
// empty name changes nothing.
func TestTheOwnNameIsWaitedForAndDiscoveryNamesIt(t *testing.T) {
o := NewOwnName()
o.Set("")
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
defer cancel()
if _, err := o.Wait(ctx); err == nil || o.Get() != "" {
t.Fatalf("an unnamed engine's wait ended with %v, name %q", err, o.Get())
}
o.Set("the-engine")
if got, err := o.Wait(context.Background()); err != nil || got != "the-engine" {
t.Fatalf("waited %q, %v", got, err)
}
if got := EngineEndpointMetadata(catalogue.Verb{Name: "composition"}, "n1", o.Get())["module"]; got != "the-engine" {
t.Fatalf("discovery names the module %q", got)
}
var none *OwnName
if _, err := none.Wait(context.Background()); err == nil || none.Get() != "" {
t.Fatalf("a session naming none waited: %v", err)
}
}
internal/link/record.go
+15
View File
@@ -33,10 +33,17 @@ type RecordKeys struct {
// Stated is the capabilities its tool runner states of the calls it answers (CapabilitiesKey), which the record says
// beside the machine's own: a module whose manifest asks for one is held off a machine whose runner does not state it.
Stated []string
// StatedBy is the fingerprint of the tool runner that stated Stated, as its PING carries it (FingerprintKey): the record
// takes both, so a reading is tied to the runner that made it (design 46 approval mark §6, lead answer A-203).
StatedBy string
}
// buildReads are the words and forms this engine reads of a manifest (novox/hq ADR 0614 §4): the reader capabilities of
// the mesh-sdk it is built with, its seats left out. Computed once: they are the build's, and never change while it runs.
// CapabilityWantParts is the capability a node-engine states in its node record when it reads the want's version, parts and
// kept-on (design 47 G §525); the assigner writes them into a node's want only while the record states it.
const CapabilityWantParts = "want-parts"
var buildReads = sync.OnceValue(func() []string {
return slices.DeleteFunc(catalogue.ManifestReads(), func(r string) bool { return strings.HasPrefix(r, catalogue.ReadSeat) })
})
@@ -125,11 +132,19 @@ func recordOf(node string, r Report, keys RecordKeys) *nodeengine.NodeRecord {
rec.Capabilities = append(rec.Capabilities, c.Name)
}
}
// A node-engine of G states that it reads the want's new fields: version, parts and where a kept build is kept (design 47 G §525).
rec.Capabilities = append(rec.Capabilities, CapabilityWantParts)
for _, c := range keys.Stated {
if c != "" && !slices.Contains(rec.Capabilities, c) {
rec.Capabilities = append(rec.Capabilities, c)
}
if c != "" {
rec.RunnerCapabilities = append(rec.RunnerCapabilities, c)
}
}
// The runner's own reading beside the merge, which stays for placement (design 46 approval mark §6, lead answer A-203).
sort.Strings(rec.RunnerCapabilities)
rec.RunnerFingerprint = keys.StatedBy
sort.Strings(rec.Capabilities)
if p.Machine != nil {
rec.NetworksSaid = true
internal/link/recovery.go
+243
View File
@@ -0,0 +1,243 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// The node agent's wait for the recovery module (design 47 G §554, §560, A-77, A-92): the ending events of one module's kept data
// are heard through an ordered consumer that begins at the last message of the subject and follows the rest, and a wait's bound
// is a key of the state bounds with its own time to live, ended by the bus's marker.
// BoundsBucket is the node-engine's state `bounds`, named by its holder as every state is.
const BoundsBucket = "mesh-host_bounds"
// RecoveryBus is a link that hears the recovery module's events and keeps a bound.
type RecoveryBus interface {
// HearEnding calls heard with the last message of the subject on the stream, then each after it, until stop is called.
HearEnding(stream, subject string, heard func([]byte)) (stop func(), err error)
// BoundCreate writes the bound key with its time to live; one that stands is left as it is. BoundWatch calls ended once the
// bus's marker says the key's time ended; BoundDelete ends it.
BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error
BoundWatch(ctx context.Context, key string, ended func()) (stop func(), err error)
BoundDelete(ctx context.Context, key string) error
}
func hearEnding(js nats.JetStreamContext, stream, subject string, heard func([]byte)) (func(), error) {
sub, err := js.Subscribe(subject, func(m *nats.Msg) { heard(m.Data) }, nats.OrderedConsumer(), nats.DeliverLast(), nats.BindStream(stream))
if err != nil {
return nil, fmt.Errorf("the stream %s cannot be followed: %w", stream, err)
}
return func() { _ = sub.Unsubscribe() }, nil
}
func boundKV(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
js, err := jetstream.New(conn)
if err != nil {
return nil, err
}
kv, err := js.KeyValue(ctx, BoundsBucket)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be bound: %w", BoundsBucket, err)
}
return kv, nil
}
func boundCreate(ctx context.Context, conn *nats.Conn, key string, value []byte, ttl time.Duration) error {
kv, err := boundKV(ctx, conn)
if err != nil {
return err
}
if _, err := kv.Create(ctx, key, value, jetstream.KeyTTL(ttl)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
return err
}
return nil
}
func boundWatch(ctx context.Context, conn *nats.Conn, key string, ended func()) (func(), error) {
kv, err := boundKV(ctx, conn)
if err != nil {
return nil, err
}
w, err := kv.Watch(ctx, key, jetstream.UpdatesOnly())
if err != nil {
return nil, err
}
go func() {
for e := range w.Updates() {
if e != nil && e.Operation() == jetstream.KeyValuePurge {
ended()
}
}
}()
return func() { _ = w.Stop() }, nil
}
func boundDelete(ctx context.Context, conn *nats.Conn, key string) error {
kv, err := boundKV(ctx, conn)
if err != nil {
return err
}
if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
func (l *natsLink) HearEnding(stream, subject string, heard func([]byte)) (func(), error) {
return hearEnding(l.js, stream, subject, heard)
}
func (l *natsLink) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error {
return boundCreate(ctx, l.conn, key, value, ttl)
}
func (l *natsLink) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) {
return boundWatch(ctx, l.conn, key, ended)
}
func (l *natsLink) BoundDelete(ctx context.Context, key string) error {
return boundDelete(ctx, l.conn, key)
}
func (b OverNATS) HearEnding(stream, subject string, heard func([]byte)) (func(), error) {
return hearEnding(b.JS, stream, subject, heard)
}
func (b OverNATS) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error {
return boundCreate(ctx, b.Conn, key, value, ttl)
}
func (b OverNATS) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) {
return boundWatch(ctx, b.Conn, key, ended)
}
func (b OverNATS) BoundDelete(ctx context.Context, key string) error {
return boundDelete(ctx, b.Conn, key)
}
var (
_ RecoveryBus = (*natsLink)(nil)
_ RecoveryBus = OverNATS{}
)
func (q *Session) recoveryBus() (RecoveryBus, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
h, ok := bus.(RecoveryBus)
if bus == nil || !ok {
return nil, ErrNoLink
}
return h, nil
}
// RecoveryEnding is the ending event of an act as the node agent reads it, the fields every ending carries.
type RecoveryEnding struct {
Act string `json:"act"`
Outcome string `json:"outcome"`
Error string `json:"error,omitempty"`
Snapshot string `json:"snapshot,omitempty"`
// Why is the hand act's words for a cancel (design 47 G A-117, §560).
Why string `json:"why,omitempty"`
}
// HearRecoveryEnding follows the events `ended` (kind "ended") or `restored` (kind "restored") of one module's kept data: the last
// said first, then each after it, until stop is called.
func (q *Session) HearRecoveryEnding(kind, module, kept string) (<-chan RecoveryEnding, func(), error) {
bus, err := q.recoveryBus()
if err != nil {
return nil, nil, err
}
stream, subject := meshrecovery.StreamEnded, meshrecovery.SubjectEnded(module, kept)
if kind == "restored" {
stream, subject = meshrecovery.StreamRestored, meshrecovery.SubjectRestored(module, kept)
}
out := make(chan RecoveryEnding, 64)
stop, err := bus.HearEnding(stream, subject, func(raw []byte) {
var ev RecoveryEnding
if json.Unmarshal(raw, &ev) == nil && ev.Act != "" {
select {
case out <- ev:
default:
}
}
})
if err != nil {
return nil, nil, err
}
return out, stop, nil
}
// KeepBound writes the bound key `<what>.<id>` of a wait with its time to live and watches it: expired is closed when the bus's
// marker says its time ended, and end deletes it.
func (q *Session) KeepBound(ctx context.Context, what, id, module string, since time.Time, ttl time.Duration) (<-chan struct{}, func(), error) {
bus, err := q.recoveryBus()
if err != nil {
return nil, nil, err
}
key := what + "." + id
value, err := json.Marshal(struct {
Module string `json:"module,omitempty"`
Since string `json:"since"`
V int64 `json:"v"`
}{module, since.UTC().Format(time.RFC3339), 1})
if err != nil {
return nil, nil, err
}
expired := make(chan struct{})
stopWatch, err := bus.BoundWatch(ctx, key, func() {
select {
case <-expired:
default:
close(expired)
}
})
if err != nil {
return nil, nil, err
}
if err := bus.BoundCreate(ctx, key, value, ttl); err != nil {
stopWatch()
return nil, nil, err
}
return expired, func() {
stopWatch()
_ = bus.BoundDelete(context.WithoutCancel(ctx), key)
}, nil
}
// CallTool calls a module's tool on this machine's own node tools with its arguments and answers its refusal as an error (design 47 G
// A-108: the catch-up of a restore point is called so, as the node-engine is granted to).
func (q *Session) CallTool(ctx context.Context, module, tool string, args map[string]any) error {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
asker, ok := bus.(ToolBus)
if bus == nil || !ok {
return ErrNoLink
}
if args == nil {
args = map[string]any{}
}
body, err := json.Marshal(args)
if err != nil {
return err
}
reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), body)
if err != nil {
return err
}
var env struct {
Error string `json:"error"`
}
if err := json.Unmarshal(reply, &env); err != nil {
return fmt.Errorf("its answer is not a tool's answer: %w", err)
}
if env.Error != "" {
return errors.New(env.Error)
}
return nil
}
internal/link/run.go
+3
View File
@@ -217,6 +217,9 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
// link, and what was answered while no link was open caught up from the router's records.
if queue.Acts != nil {
if s, ok := link.(EngineSeats); ok {
if o, named := link.(OwnNamed); named {
o.UseOwnName(queue.Own)
}
s.ServeEngineSeat(queue.Acts.Call)
// And what it reads of a manifest, said once for this link (novox/hq ADR 0618 §4).
queue.SayReads()
internal/link/running.go
+188
View File
@@ -0,0 +1,188 @@
package link
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
)
// The node seat's event running, said by every node-engine for each module it saw running healthy, is kept on the stream
// SEAT_node-engine_running, the last message of each subject with no age limit (design 47 G §199, §260). The node reads it
// at each start through an ordered consumer it makes for that one read (§556), then follows a durable consumer of its own
// from its acknowledged position: what the other nodes run is known from the last message of each subject however long ago it
// was said, never from a have key or a call.
// RunningBus is a link that hears the seat's event running of every node.
type RunningBus interface {
// FirstRunning calls first with the last message of every subject, through an ordered consumer made for this read, and
// answers when it has read them all.
FirstRunning(ctx context.Context, first func([]byte)) error
// HearRunning calls heard with each message after the durable consumer's acknowledged position (a new consumer begins at
// the last message of each subject), until stop is called or the link ends.
HearRunning(consumer string, heard func([]byte)) (stop func(), err error)
}
// RunningConsumer is the name of this node's durable consumer of the stream.
func RunningConsumer(node string) string { return "node-engine-" + node + "-running" }
func (l *natsLink) FirstRunning(ctx context.Context, first func([]byte)) error {
sub, err := l.js.SubscribeSync(nodeengine.SubjectRunningAll, nats.OrderedConsumer(), nats.DeliverLastPerSubject(),
nats.BindStream(nodeengine.StreamRunning))
if err != nil {
return fmt.Errorf("the stream %s cannot be read: %w", nodeengine.StreamRunning, err)
}
defer sub.Unsubscribe()
info, err := sub.ConsumerInfo()
if err != nil {
return fmt.Errorf("the stream %s cannot be read: %w", nodeengine.StreamRunning, err)
}
for total, got := info.Delivered.Consumer+info.NumPending, uint64(0); got < total; got++ {
msg, err := sub.NextMsgWithContext(ctx)
if err != nil {
return fmt.Errorf("reading the last running of each subject (%d of %d): %w", got, total, err)
}
first(msg.Data)
}
return nil
}
func (l *natsLink) HearRunning(consumer string, heard func([]byte)) (func(), error) {
sub, err := l.js.Subscribe(nodeengine.SubjectRunningAll, func(m *nats.Msg) {
heard(m.Data)
_ = m.Ack()
}, nats.Durable(consumer), nats.BindStream(nodeengine.StreamRunning), nats.DeliverLastPerSubject(), nats.AckExplicit(),
nats.ManualAck())
if err != nil {
return nil, fmt.Errorf("the stream %s cannot be followed: %w", nodeengine.StreamRunning, err)
}
return func() { _ = sub.Unsubscribe() }, nil
}
var _ RunningBus = (*natsLink)(nil)
// FollowRunning hands every node's running event to heard: the last of each subject first, with readEnded called after them
// (until then the caller acts on none, §556), then each after them, until ctx ends. An error is the stream or the link not
// being there, which the caller tries again.
func (q *Session) FollowRunning(ctx context.Context, heard func(nodeengine.Running), readEnded func()) error {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
h, ok := bus.(RunningBus)
if bus == nil || !ok {
return ErrNoLink
}
decode := func(raw []byte) {
var ev nodeengine.Running
if json.Unmarshal(raw, &ev) == nil && ev.Node != "" && ev.Module != "" {
heard(ev)
}
}
if err := h.FirstRunning(ctx, decode); err != nil {
return err
}
stop, err := h.HearRunning(RunningConsumer(q.Membership.Node), decode)
if err != nil {
return err
}
defer stop()
readEnded()
<-ctx.Done()
return nil
}
// OverNATS reads the stream as the link's session does, over its connection.
func (b OverNATS) FirstRunning(ctx context.Context, first func([]byte)) error {
return (&natsLink{js: b.JS}).FirstRunning(ctx, first)
}
// HearRunning follows the stream over the connection (see natsLink.HearRunning).
func (b OverNATS) HearRunning(consumer string, heard func([]byte)) (func(), error) {
return (&natsLink{js: b.JS}).HearRunning(consumer, heard)
}
var _ RunningBus = OverNATS{}
// The builder's event build.ended (design 47 G A-293, as the assigner hears it, A-181, §556): the builder's stream of build
// ends keeps the last message of each module and fingerprint, and the node reads it whole at each start, then follows it, through
// one ordered consumer it makes for the run: what ended, and a failed build's words, are known from the stream however long ago
// they were said, never from the builder's builds. The engine's manifest consumes `mesh-delivery-builder.build.ended.**`.
// BuildEndsStream is the builder's stream of build ends as the bus names it, BuildEndsSubject every event it carries.
var (
BuildEndsStream = catalogue.StreamBusName(builds.BuildsStream)
BuildEndsSubject = "mesh.stream.mesh-delivery-builder.event." + builds.BuildEndedEvent + ".>"
)
// BuildEndsBus is a link that hears the builder's build ends.
type BuildEndsBus interface {
// HearBuildEnds calls heard with the last message of every subject, then readEnded once they are all read, then heard with
// each after them, until ctx ends (nil) or the stream is lost (its error).
HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error
}
func hearBuildEnds(ctx context.Context, js nats.JetStreamContext, heard func([]byte), readEnded func()) error {
sub, err := js.SubscribeSync(BuildEndsSubject, nats.OrderedConsumer(), nats.DeliverLastPerSubject(), nats.BindStream(BuildEndsStream))
if err != nil {
return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err)
}
defer sub.Unsubscribe()
info, err := sub.ConsumerInfo()
if err != nil {
return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err)
}
total := info.Delivered.Consumer + info.NumPending
for got := uint64(0); ; got++ {
if got == total {
readEnded()
}
msg, err := sub.NextMsgWithContext(ctx)
if err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("the stream %s is not followed any more (%d read): %w", BuildEndsStream, got, err)
}
heard(msg.Data)
}
}
func (l *natsLink) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error {
return hearBuildEnds(ctx, l.js, heard, readEnded)
}
// HearBuildEnds reads the stream as the link's session does, over its connection.
func (b OverNATS) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error {
return hearBuildEnds(ctx, b.JS, heard, readEnded)
}
var (
_ BuildEndsBus = (*natsLink)(nil)
_ BuildEndsBus = OverNATS{}
)
// FollowBuildEnds hands every build end the builder said to heard: the last of each module and fingerprint first, with
// readEnded called after them (until then the caller acts on none), then each after them, until ctx ends. A body that is no
// end of a module at a fingerprint is passed over. An error is the stream or the link not being there, which the caller tries
// again.
func (q *Session) FollowBuildEnds(ctx context.Context, heard func(builds.BuildEnded), readEnded func()) error {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
h, ok := bus.(BuildEndsBus)
if bus == nil || !ok {
return ErrNoLink
}
return h.HearBuildEnds(ctx, func(raw []byte) {
var e builds.BuildEnded
if json.Unmarshal(raw, &e) == nil && e.Module != "" && e.Fingerprint != "" {
heard(e)
}
}, readEnded)
}
internal/link/session.go
+120 -9
View File
@@ -12,9 +12,13 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/operatorask"
)
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this
@@ -23,6 +27,13 @@ import (
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here
// receives, orders or reconciles a whole declaration.
type Session struct {
// Own is the module that is this node-engine (§527, §634): its discovery, finding writer and vault login names read it.
// Nil names none, as a test's session does.
Own *OwnName
// haveSay guards haveSaidFn and haveGoneFn (SetHaveSaid).
haveSay sync.Mutex
haveSaidFn func(module string, h nodeengine.HaveByNodeModuleValue)
haveGoneFn func(module string)
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
Fallback func() *nodeengine.LoginFallback
// linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs).
@@ -86,6 +97,8 @@ type Session struct {
// runnerStated is the capabilities this machine's tool runner last stated of the calls it answers (askRunner), said in the
// node's record. Held under saying.
runnerStated []string
// runnerFingerprint is the fingerprint of the runner that stated runnerStated (FingerprintKey). Held under saying.
runnerFingerprint string
// saying is held while a facts event is noted, so the haves are written in the order the facts are made.
saying sync.Mutex
@@ -100,9 +113,14 @@ type Session struct {
}
// restating takes RestateReads one at a time.
restating sync.Mutex
// retrying counts the says being said again on the link open (sayAgain).
retrying sync.WaitGroup
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
haves haveState
// manifestFn names the manifest of the build a module runs, for the event running (SetManifestOf), under manifest.
manifest sync.Mutex
manifestFn func(module, fp string) string
// healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone
// writes it, one look at a time.
healthWritten *NodeHealth
@@ -177,6 +195,10 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
bus := q.bus
q.mu.Unlock()
if bus == nil {
// With no link open the facts are still taken in memory, so the node's own record is known to what composes before the
// bus is read (design 47 G A-120); nothing is written.
r.Node = q.Membership.Node
q.noteApply(r)
return false
}
r.Node = q.Membership.Node
@@ -190,6 +212,19 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
return true
}
// OwnRecord is the record of this node the node-engine states of itself (tunnel key, sealing key, capabilities), from the facts it
// last took, held in memory (design 47 G A-120); false before any.
func (q *Session) OwnRecord() (nodeengine.NodeRecord, bool) {
q.init()
s := &q.haves
s.mu.Lock()
defer s.mu.Unlock()
if s.node == nil || s.node.Record == nil {
return nodeengine.NodeRecord{}, false
}
return *s.node.Record, true
}
// FactsAsked is the mesh asking this node to say its facts again: run on the link open now.
func (q *Session) FactsAsked(ctx context.Context) {
q.init()
@@ -345,6 +380,12 @@ func SeatToolSubject(seat, verb, node string) string {
// AskSeat asks the holder of a seat on this node a verb, over the link open now, and answers its result, or its refusal
// as an error. The one call the node agent makes of another part: the backup before a data module moves (ADR 0429 §14).
func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, within time.Duration) (json.RawMessage, error) {
return q.AskSeatOn(ctx, q.Membership.Node, seat, verb, args, within)
}
// AskSeatOn asks the holder of a node seat on another node a verb: the build seat of the node a want's `kept-on` names
// gives the kept build (design 47 G section 526).
func (q *Session) AskSeatOn(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error) {
q.init()
q.mu.Lock()
bus := q.bus
@@ -362,7 +403,7 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with
}
asking, cancel := context.WithTimeout(ctx, within)
defer cancel()
reply, err := asker.Ask(asking, SeatToolSubject(seat, verb, q.Membership.Node), body)
reply, err := asker.Ask(asking, SeatToolSubject(seat, verb, node), body)
if err != nil {
return nil, fmt.Errorf("%s.%s could not be asked: %w", seat, verb, err)
}
@@ -394,7 +435,16 @@ func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) {
if err != nil {
return nil, err
}
return asking.RouterRecord(key)
// The bucket is the one of the seat's holder on the seat list, found by mesh-sdk's operatorask (design 47 G A-101).
bucket, err := operatorask.Bucket(func(call string, body any) (json.RawMessage, error) {
ctx, cancel := context.WithTimeout(context.Background(), VaultWithin)
defer cancel()
return q.AskMeshSeat(ctx, meshmoduleassigner.Seat, meshmoduleassigner.VerbAssignments, body)
}, operatorask.Contract{Seat: occ.Seat, StateAsks: occ.StateAsks})
if err != nil {
return nil, err
}
return asking.RouterRecord(bucket, key)
}
// StatedReads is what this engine states it reads of a manifest and when it made that statement, at its start or at the
@@ -463,22 +513,83 @@ func (q *Session) SayReads() {
Node: q.Membership.Node, ManifestReads: manifestReads(), At: time.Now().UTC().Format(time.RFC3339Nano), Why: why})
}
// SayEngine says one of the node seat's events over the link open now. With none open, or a publish that failed past the
// bus's retry bound, it is kept and said at the next link's opening (SayReads; novox/hq ADR 0668 §4, issue 1966): an
// event a reader waits on is never only a line in this engine's log.
// sayEngineOn says one of the node seat's events on a subject of its own (running, composed: a module in it), over the link
// open now; with none open it is not said.
func (q *Session) sayEngineOn(subject, event string, body any) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if sayer, ok := bus.(interface{ SayEngineOn(string, any) error }); ok {
if err := sayer.SayEngineOn(subject, body); err != nil {
q.Say("could not say the node seat's " + event + ": " + err.Error())
}
}
}
// SayEngine says one of the node seat's events over the link open now. A publish the bus did not take is said again on
// that link until it takes it (sayAgain; design 47 G A-292, §590); with no link open, or a link lost meanwhile, or a
// refusal the bus answered, it is kept and said at the next link's opening (SayReads; novox/hq ADR 0668 §4, issue 1966):
// an event a reader waits on is never only a line in this engine's log.
func (q *Session) SayEngine(event string, body any) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
sayer, ok := bus.(interface{ SayEngine(string, any) error })
var err error
sayer, ok := bus.(engineSayer)
if ok {
if err = sayer.SayEngine(event, body); err == nil {
err := sayer.SayEngine(event, body)
if err == nil {
return
}
if sayableAgain(err) {
// A say the bus did not take is said again on this link until the bus takes it (design 47 G A-292, §590):
// off the caller, so a link's setup and a restatement never wait on it.
q.Say("could not say the node seat's " + event + ", said again on this link until the bus takes it: " + err.Error())
q.retrying.Add(1)
go q.sayAgain(bus, sayer, event, body)
return
}
q.Say("could not say the node seat's " + event + ", said again at the next link: " + err.Error())
}
q.holdUnsaid(event, body)
}
// engineSayer is a link that says the node seat's events.
type engineSayer interface{ SayEngine(string, any) error }
// sayableAgain is an error the SDK's bus retry asks again on: the bus took nothing (no responders, a deadline). An error
// the bus answered is not asked again on this link.
func sayableAgain(err error) bool { return busretry.Undelivered(err) || busretry.TimedOut(err) }
// sayAgain says event again through the link's bus retry for as long as bus is the link open and the bus did not take it,
// writing each failed attempt to the engine's journal with the event's name (A-292). A link lost meanwhile keeps the
// event for the next link (holdUnsaid).
func (q *Session) sayAgain(bus Bus, sayer engineSayer, event string, body any) {
defer q.retrying.Done()
for q.linkedTo(bus) {
err := sayer.SayEngine(event, body)
if err == nil {
return
}
if !sayableAgain(err) {
q.Say("could not say the node seat's " + event + ", said again at the next link: " + err.Error())
break
}
q.Say("could not say the node seat's " + event + " again, said again on this link until the bus takes it: " + err.Error())
}
q.holdUnsaid(event, body)
}
// linkedTo says bus is the link open now.
func (q *Session) linkedTo(bus Bus) bool {
q.mu.Lock()
defer q.mu.Unlock()
return q.bus != nil && q.bus == bus
}
// holdUnsaid keeps event for the next link's SayReads: no link is open, or the bus answered it with a refusal.
func (q *Session) holdUnsaid(event string, body any) {
q.reads.Lock()
if q.reads.unsaid == nil {
q.reads.unsaid = map[string]any{}
@@ -526,5 +637,5 @@ func (q *Session) askRunner(ctx context.Context, bus Bus) {
stated = append(stated, c)
}
}
q.runnerStated = stated
q.runnerStated, q.runnerFingerprint = stated, strings.TrimSpace(md[FingerprintKey])
}
internal/link/session_test.go
+65
View File
@@ -11,6 +11,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"github.com/nats-io/nats.go"
)
@@ -24,6 +26,37 @@ func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
func (l *quietLink) Lost() <-chan error { return l.lost }
func (l *quietLink) Close() {}
// runningLink hears the node seat's running events: the last of each subject first, then the durable consumer's.
type runningLink struct {
*quietLink
first, after [][]byte
}
func (l *runningLink) FirstRunning(_ context.Context, first func([]byte)) error {
for _, raw := range l.first {
first(raw)
}
return nil
}
func (l *runningLink) HearRunning(_ string, heard func([]byte)) (func(), error) {
for _, raw := range l.after {
heard(raw)
}
return func() {}, nil
}
func (l *runningLink) HearBuildEnds(_ context.Context, heard func([]byte), readEnded func()) error {
for _, raw := range l.first {
heard(raw)
}
readEnded()
for _, raw := range l.after {
heard(raw)
}
return nil
}
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
t.Helper()
public, private, err := ed25519.GenerateKey(nil)
@@ -46,6 +79,38 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
if s.Linked() || s.SayFacts(context.Background(), Report{}) {
t.Fatal("facts were said on a link that was let go")
}
// Design 47 G §199, §556: with no link the running events are not followed; with one, the last message of each subject
// comes first, the read ends, then the durable consumer's, and a body that is no running event is passed over.
if err := s.FollowRunning(context.Background(), func(nodeengine.Running) {}, func() {}); !errors.Is(err, ErrNoLink) {
t.Fatalf("followed with no link: %v", err)
}
rl := &runningLink{quietLink: newQuietLink(),
first: [][]byte{[]byte(`{"node":"a","module":"x","fingerprint":"f1"}`), []byte(`not json`), []byte(`{"module":"x"}`)},
after: [][]byte{[]byte(`{"node":"b","module":"y","stopped":true}`)}}
s.Attach(context.Background(), rl)
var order []string
ctx, stop := context.WithCancel(context.Background())
err := s.FollowRunning(ctx, func(ev nodeengine.Running) { order = append(order, ev.Node+"/"+ev.Module) },
func() { order = append(order, "read"); stop() })
if err != nil || strings.Join(order, " ") != "a/x b/y read" {
t.Fatalf("followed %v: %v, want a/x b/y read", order, err)
}
// Design 47 G A-293: the builder's build ends likewise, the last of each subject first, then the read's end, then the
// rest, a body that is no end of a module at a fingerprint passed over.
if err := (&Session{Membership: m}).FollowBuildEnds(context.Background(), func(builds.BuildEnded) {}, func() {}); !errors.Is(err, ErrNoLink) {
t.Fatalf("build ends followed with no link: %v", err)
}
rl.first = [][]byte{[]byte(`{"module":"x","fingerprint":"f1","outcome":"failed","error":"no go"}`), []byte(`not json`), []byte(`{"module":"x"}`)}
rl.after = [][]byte{[]byte(`{"module":"y","fingerprint":"f2","outcome":"done","branch":"b"}`)}
order = nil
err = s.FollowBuildEnds(context.Background(), func(e builds.BuildEnded) {
order = append(order, e.Module+"@"+e.Fingerprint+":"+e.Outcome+":"+e.Error+":"+e.Branch)
}, func() { order = append(order, "read") })
if err != nil || strings.Join(order, " ") != "x@f1:failed:no go: read y@f2:done::b" {
t.Fatalf("build ends followed %v: %v", order, err)
}
}
// A link opening says the node's facts, off the link's own loop.
internal/link/withheld.go
+47
View File
@@ -0,0 +1,47 @@
package link
import (
"context"
"encoding/json"
)
// **The composition names the withheld subjects** (design 46 approval mark §6, lead answer A-239): the node-engine reads them
// through the seat mesh-grants' verb withheld, never from the grants part's state; when the verb does not answer, it says so.
// GrantsSeat is the seat of the grants part, whose verb withheld names the marked subjects withheld from a node's runtime.
const GrantsSeat = "mesh-grants"
// WithheldUnknown is what a composition says when the grants part does not answer withheld (design 46 approval mark §6).
const WithheldUnknown = "withheld unknown: the grants part does not answer"
// WithheldSubject is one marked subject withheld from this node's runtime, and why.
type WithheldSubject struct {
Subject string `json:"subject"`
Why string `json:"why"`
}
// WithheldOf is what the grants part withholds of module on this node, asked through its verb withheld: the subjects, or,
// when the verb does not answer or answers what is not its answer, WithheldUnknown as unknown.
func (q *Session) WithheldOf(ctx context.Context, module string) (subjects []WithheldSubject, unknown string) {
raw, err := q.askCalled(ctx, GrantsSeat, "withheld", map[string]string{"node": q.Membership.Node}, VaultWithin,
"the grants part's withheld")
if err != nil {
return nil, WithheldUnknown
}
var answer struct {
Node string `json:"node"`
Modules []struct {
Module string `json:"module"`
Subjects []WithheldSubject `json:"subjects"`
} `json:"modules"`
}
if json.Unmarshal(raw, &answer) != nil || answer.Node != q.Membership.Node {
return nil, WithheldUnknown
}
for _, m := range answer.Modules {
if m.Module == module {
subjects = append(subjects, m.Subjects...)
}
}
return subjects, ""
}
internal/link/witnessing.go
+3
View File
@@ -56,6 +56,9 @@ func (l *natsLink) Ping(ctx context.Context, service, id string) error {
const (
ToolRunnerService = "node-tools"
CapabilitiesKey = "capabilities"
// FingerprintKey is the metadata key of its PING it states its own fingerprint under, beside CapabilitiesKey (design 46
// approval mark §6, lead answer A-203).
FingerprintKey = "fingerprint"
)
// Stater is a link that can ask what a service's instance states of itself in its PING: its metadata.
internal/nodegrant/testdata/node.json
Vendored
+20 -9
View File
@@ -3,8 +3,8 @@
"$JS.ACK.NODES.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_composed-NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_assignments.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-builder_builds.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_candidates.>",
@@ -13,17 +13,21 @@
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_verdicts.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_acts.*.$KV.mesh-host_acts.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_data.*.$KV.mesh-host_data.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_findings.*.$KV.mesh-host_findings.machine.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_findings.*.$KV.mesh-host_findings.>",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_health.*.$KV.mesh-host_health.NODE",
"$JS.API.CONSUMER.CREATE.KV_mesh-host_packages.*.$KV.mesh-host_packages.NODE",
"$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_provisions.*.$KV.mesh-provisioner_provisions.NODE.>",
"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.*.node",
"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.NODE.>",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-NODE.*.$O.declaration-files-NODE.C.*",
"$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked",
"$JS.API.CONSUMER.CREATE.SEAT_node-engine_running",
"$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked.>",
"$JS.API.CONSUMER.CREATE.SEAT_node-engine_running.>",
"$JS.API.CONSUMER.DELETE.KV_composed-NODE.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_assignments.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.>",
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_candidates.>",
@@ -38,11 +42,14 @@
"$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_provisions.>",
"$JS.API.CONSUMER.DELETE.KV_node-engine_have.>",
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-NODE.*",
"$JS.API.CONSUMER.DELETE.SEAT_mesh-conditions_findings-asked.>",
"$JS.API.CONSUMER.DELETE.SEAT_node-engine_running.>",
"$JS.API.CONSUMER.INFO.SEAT_node-engine_running.>",
"$JS.API.CONSUMER.INFO.NODES.NODE",
"$JS.API.DIRECT.GET.KV_composed-NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_assignments.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-builder_builds.>",
"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_candidates.>",
@@ -51,7 +58,7 @@
"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_verdicts.>",
"$JS.API.DIRECT.GET.KV_mesh-host_acts.$KV.mesh-host_acts.NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-host_data.$KV.mesh-host_data.NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-host_findings.$KV.mesh-host_findings.machine.NODE.>",
"$JS.API.DIRECT.GET.KV_mesh-host_findings.$KV.mesh-host_findings.>",
"$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.NODE",
"$JS.API.DIRECT.GET.KV_mesh-host_packages.$KV.mesh-host_packages.NODE",
"$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.NODE.>",
@@ -60,8 +67,8 @@
"$JS.API.DIRECT.GET.OBJ_declaration-files-NODE.$O.declaration-files-NODE.M.*",
"$JS.API.STREAM.INFO.KV_composed-NODE",
"$JS.API.STREAM.INFO.KV_mesh-config_resolved",
"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates",
"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_assignments",
"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates",
"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants",
"$JS.API.STREAM.INFO.KV_mesh-delivery-builder_builds",
"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_candidates",
@@ -75,10 +82,12 @@
"$JS.API.STREAM.INFO.KV_mesh-host_packages",
"$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions",
"$JS.API.STREAM.INFO.KV_node-engine_have",
"$JS.API.STREAM.INFO.SEAT_mesh-conditions_findings-asked",
"$JS.API.STREAM.INFO.SEAT_node-engine_running",
"$JS.FC.KV_composed-NODE.>",
"$JS.FC.KV_mesh-config_resolved.>",
"$JS.FC.KV_mesh-delivery-assigner_candidates.>",
"$JS.FC.KV_mesh-delivery-assigner_assignments.>",
"$JS.FC.KV_mesh-delivery-assigner_candidates.>",
"$JS.FC.KV_mesh-delivery-assigner_wants.>",
"$JS.FC.KV_mesh-delivery-builder_builds.>",
"$JS.FC.KV_mesh-delivery-indexer_candidates.>",
@@ -96,7 +105,7 @@
"$KV.composed-NODE.>",
"$KV.mesh-host_acts.NODE.>",
"$KV.mesh-host_data.NODE.>",
"$KV.mesh-host_findings.machine.NODE.>",
"$KV.mesh-host_findings.>",
"$KV.mesh-host_health.NODE",
"$KV.mesh-host_packages.NODE",
"$KV.node-engine_have.NODE.>",
@@ -106,12 +115,14 @@
"$SRV.PING.>",
"$SRV.PING.node-tools.NODE",
"mesh.control.NODE.>",
"mesh.seat.mesh-catalog.call.entry.node~NODE",
"mesh.seat.mesh-catalog.call.seat-definitions.node~NODE",
"mesh.seat.node-backup.call.backed-up.NODE.node~NODE",
"mesh.seat.node-backup.call.now.NODE.node~NODE",
"mesh.seat.node-backup.tool.backed-up.NODE",
"mesh.seat.node-backup.tool.now.NODE",
"mesh.seat.node-engine.event.*.NODE"
"mesh.seat.node-engine.event.*.NODE",
"mesh.stream.mesh-host.event.*.NODE"
],
"PublishDeny": null,
"Subscribe": [
internal/processbuilds/processbuilds.go
+140
View File
@@ -0,0 +1,140 @@
// Package processbuilds reads the build each process of this node was started with (design 47, *The fingerprint handover*):
// the value MESH_FINGERPRINT the node-engine handed a daemon or a container at its start, read from the running process
// itself, never from what the declaration would hand it now. A process started before the node-engine handed the value,
// or one whose value cannot be read, says why its build is unknown instead of a fingerprint.
package processbuilds
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
// Var is the environment value the build is handed in.
const Var = "MESH_FINGERPRINT"
// Runner runs a command and answers its output.
type Runner func(ctx context.Context, name string, args ...string) (string, error)
// Reader reads the running processes of a node.
type Reader struct {
Node string
Run Runner
// Environ reads a process's environment by its pid; nil reads /proc/<pid>/environ.
Environ func(pid int) ([]byte, error)
}
func (r Reader) environ(pid int) ([]byte, error) {
if r.Environ != nil {
return r.Environ(pid)
}
return os.ReadFile(fmt.Sprintf("/proc/%d/environ", pid))
}
// Read answers the build of every daemon and container the declaration holds, narrowed to module when it is not empty,
// ordered by module and process.
func (r Reader) Read(ctx context.Context, d *declaration.Declaration, module string) nodeengine.ProcessBuilds {
out := nodeengine.ProcessBuilds{Node: r.Node, Processes: []nodeengine.ProcessBuild{}}
if d == nil {
out.Error = "this node keeps no declaration of what it runs"
return out
}
for _, res := range d.Resources {
owner := res.DeclaredModule()
if owner == "" {
owner, _ = declaration.ModuleGuess(res.Identity())
}
if module != "" && owner != module {
continue
}
switch p := res.(type) {
case *declaration.Process:
out.Processes = append(out.Processes, r.daemon(ctx, owner, p.Name))
case *declaration.Container:
out.Processes = append(out.Processes, r.container(ctx, owner, p.Name))
}
}
sort.SliceStable(out.Processes, func(i, j int) bool {
a, b := out.Processes[i], out.Processes[j]
if a.Module != b.Module {
return a.Module < b.Module
}
return a.Process < b.Process
})
return out
}
func (r Reader) daemon(ctx context.Context, module, name string) nodeengine.ProcessBuild {
unit := name + ".service"
b := nodeengine.ProcessBuild{Module: module, Process: unit, Kind: nodeengine.ProcessBuildKindService}
said, err := r.Run(ctx, "systemctl", "show", "--property=MainPID,ActiveEnterTimestamp", "--", unit)
if err != nil {
b.Unknown = "the unit cannot be read: " + err.Error()
return b
}
props := map[string]string{}
for _, line := range strings.Split(said, "\n") {
if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok {
props[k] = v
}
}
pid, _ := strconv.Atoi(props["MainPID"])
if pid <= 0 {
b.Unknown = "the unit runs no process now"
return b
}
if t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", props["ActiveEnterTimestamp"]); err == nil {
b.Started = &t
}
raw, err := r.environ(pid)
if err != nil {
b.Unknown = "its environment cannot be read: " + err.Error()
return b
}
b.Fingerprint, b.Unknown = fingerprintIn(strings.Split(string(raw), "\x00"))
return b
}
func (r Reader) container(ctx context.Context, module, name string) nodeengine.ProcessBuild {
b := nodeengine.ProcessBuild{Module: module, Process: name, Kind: nodeengine.ProcessBuildKindContainer}
var said string
var err error
for _, cri := range []string{"docker", "podman"} {
if said, err = r.Run(ctx, cri, "container", "inspect", "--format", "{{json .Config.Env}}\t{{.State.StartedAt}}", name); err == nil {
break
}
}
if err != nil {
b.Unknown = "the container cannot be read: " + err.Error()
return b
}
env, started, _ := strings.Cut(strings.TrimSpace(said), "\t")
var vars []string
if err := json.Unmarshal([]byte(env), &vars); err != nil {
b.Unknown = "its environment cannot be read: " + err.Error()
return b
}
if t, err := time.Parse(time.RFC3339Nano, started); err == nil {
b.Started = &t
}
b.Fingerprint, b.Unknown = fingerprintIn(vars)
return b
}
// fingerprintIn is the value of the handed variable in a process's environment, or why there is none.
func fingerprintIn(vars []string) (fingerprint, unknown string) {
for _, v := range vars {
if value, ok := strings.CutPrefix(v, Var+"="); ok && value != "" {
return value, ""
}
}
return "", "started before the node-engine handed " + Var + ", or without it"
}
internal/processbuilds/processbuilds_test.go
+70
View File
@@ -0,0 +1,70 @@
package processbuilds
import (
"context"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
// Design 47, *The fingerprint handover*: the build of a daemon is read from its running process's environment and of a
// container from its configuration, narrowed by module; one started without the value, one not running and one the
// runtime cannot read each say why their build is unknown, never a guessed fingerprint.
func TestProcessBuildsAreReadFromTheRunningProcesses(t *testing.T) {
fp := "src2:" + strings.Repeat("a", 64)
d := &declaration.Declaration{Resources: []declaration.Resource{
&declaration.Process{ID: "web.d", Type: declaration.TypeProcess, Module: "web", Name: "web-d"},
&declaration.Process{ID: "old.d", Type: declaration.TypeProcess, Module: "old", Name: "old-d"},
&declaration.Process{ID: "down.d", Type: declaration.TypeProcess, Module: "down", Name: "down-d"},
&declaration.Container{ID: "db.c", Type: declaration.TypeContainer, Module: "db", Name: "db-c"},
&declaration.Container{ID: "gone.c", Type: declaration.TypeContainer, Module: "gone", Name: "gone-c"},
&declaration.Directory{ID: "web.dir", Type: declaration.TypeDirectory, Module: "web"},
}}
run := func(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "systemctl" && args[len(args)-1] == "web-d.service":
return "MainPID=11\nActiveEnterTimestamp=Mon 2026-11-09 10:00:00 UTC\n", nil
case name == "systemctl" && args[len(args)-1] == "old-d.service":
return "MainPID=12\nActiveEnterTimestamp=\n", nil
case name == "systemctl":
return "MainPID=0\nActiveEnterTimestamp=\n", nil
case name == "docker" && args[len(args)-1] == "db-c":
return `["A=1","MESH_FINGERPRINT=` + fp + `"]` + "\t2026-11-09T10:00:00.5Z\n", nil
}
return "", errors.New("no such container")
}
environ := func(pid int) ([]byte, error) {
if pid == 11 {
return []byte("A=1\x00MESH_FINGERPRINT=" + fp + "\x00"), nil
}
return []byte("A=1\x00"), nil
}
r := Reader{Node: "n1", Run: run, Environ: environ}
got := r.Read(context.Background(), d, "")
by := map[string]string{}
for _, p := range got.Processes {
by[p.Module] = p.Fingerprint + "|" + p.Unknown
}
want := map[string]string{"web": fp + "|", "db": fp + "|",
"old": "|started before the node-engine handed MESH_FINGERPRINT, or without it",
"down": "|the unit runs no process now", "gone": "|the container cannot be read: no such container"}
if len(by) != len(want) || got.Node != "n1" {
t.Fatalf("read %v, want %v", by, want)
}
for m, w := range want {
if by[m] != w {
t.Errorf("%s: %q, want %q", m, by[m], w)
}
}
if got.Processes[0].Module != "db" || got.Processes[len(got.Processes)-1].Module != "web" || got.Processes[len(got.Processes)-1].Started == nil {
t.Errorf("order or start time wrong: %+v", got.Processes)
}
if one := r.Read(context.Background(), d, "web"); len(one.Processes) != 1 || one.Processes[0].Process != "web-d.service" {
t.Errorf("narrowed by module: %+v", one.Processes)
}
if none := r.Read(context.Background(), nil, ""); none.Error == "" || none.Processes == nil {
t.Errorf("no declaration: %+v", none)
}
}
internal/registryforms/registryforms.go
+3 -3
View File
@@ -4,9 +4,9 @@
package registryforms
import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
containerimageregistry "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
ociartifactregistry "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
containerimageregistry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
ociartifactregistry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
)
// Forms reads, records and routes a reference in the registry seat's form and the older artifact-store one.
internal/seatwait/entrynotfound.go
+33
View File
@@ -0,0 +1,33 @@
package seatwait
import (
"context"
"fmt"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// EntryNotFoundKind is the finding a part writes when the catalog half holds no entry of a fingerprint the part holds a want or a
// running for (design 47 G §614, A-15, A-117): scope module, its id the fingerprint escaped as a name.
const EntryNotFoundKind = "entry-not-found"
// EntryNotFoundID is the finding's id for a fingerprint.
func EntryNotFoundID(fp string) string { return keys.EscapeName(fp) }
// SayEntryNotFound writes the finding for fp when notFound, and clears it when no longer: the entry answered later, or nothing the
// node holds names the fingerprint any more. A write or a clear that fails is returned for the caller to say.
func SayEntryNotFound(ctx context.Context, w Writer, node, observer, fp string, notFound bool, now func() time.Time) error {
id := EntryNotFoundID(fp)
if !notFound {
return w.Clear(ctx, scopeModule, id, EntryNotFoundKind, "the entry is answered, or nothing the node holds names its fingerprint")
}
return w.Write(ctx, condseat.Finding{Observer: observer, Kind: EntryNotFoundKind, Scope: scopeModule, ID: id,
Severity: condseat.FindingSeverityWarning, Source: EntryNotFoundKind,
Subject: condseat.Subject{Scope: scopeModule, ID: observer, Machine: node},
Summary: fmt.Sprintf("the catalog half holds no entry of the build %s, so %s composes nothing new of it", fp, node),
Headline: fmt.Sprintf("no entry of the build %s on %s", fp, node),
Explanation: "A want or a running names this build and the catalog half answered that it holds no entry of it. The finding clears when the entry is answered or nothing the node holds names the build.",
At: now()})
}
internal/seatwait/held.go
+109
View File
@@ -0,0 +1,109 @@
// HeldKeeper keeps the node-engine's finding `module-held`: a module the node leaves out of its pass because its
// claim cannot be held (its seat's definition missing, removed or refused, a claim another holder keeps, a credential the
// vault refuses) is no silent leave-out. The finding names the node, the module and why, and the condition keeper keeps the
// condition (novox/hq ADR 0564 §2) until the module is composed again or is no longer wanted on the node, when the finding
// is cleared (design 47, *A claim problem holds its module, never the node's pass*; test 55).
package seatwait
import (
"context"
"fmt"
"sort"
"strings"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// HeldKind is the finding of a held module; its id is `<module>.<node>`.
const HeldKind = "module-held"
// HeldKeeper writes one finding for each module the node holds, and clears it when the module is no longer held. What it
// has written is what its writer holds (design 47 G A-29, A-38, A-40): the findings an earlier run of the engine left are
// seeded into it at its Start, so a module held then and composed while the engine was away loses its condition at the
// first Reconcile, and the keeper keeps no set of its own.
type HeldKeeper struct {
Node string
Observer string
// ObserverOf, when set, answers the observer at each write (see Waiter).
ObserverOf func() string
Writer Writer
// Ctx is the run's context for the writer's acts; nil is the background.
Ctx context.Context
// Now is the clock; nil is time.Now.
Now func() time.Time
// Say tells the node's journal; nil says nothing.
Say func(string)
}
// HeldID is the finding's id for a module on a node.
func HeldID(module, node string) string { return module + "." + node }
// Reconcile makes the findings those of held (each module held and why): one is written for a module not yet written or
// held for other words, and one is cleared for a module no longer held, composed or no longer wanted here. A write or a
// clear that fails is said and tried again at the next Reconcile.
func (k *HeldKeeper) Reconcile(held map[string]string) {
ctx := k.Ctx
if ctx == nil {
ctx = context.Background()
}
written := map[string]condseat.Finding{} // by module, what the writer holds of this node's held modules
for _, f := range k.Writer.Held() {
if f.Kind != HeldKind || f.Scope != scopeModule {
continue
}
if module, ok := strings.CutSuffix(f.ID, "."+k.Node); ok {
written[module] = f
}
}
names := make([]string, 0, len(written))
for m := range written {
names = append(names, m)
}
sort.Strings(names)
for _, m := range names {
if _, still := held[m]; still {
continue
}
if err := k.Writer.Clear(ctx, scopeModule, written[m].ID, HeldKind, "the module is composed again or is no longer wanted on the node"); err != nil {
k.say(fmt.Sprintf("the finding that %s is held on %s cannot be cleared: %v", m, k.Node, err))
}
}
names = names[:0]
for m := range held {
names = append(names, m)
}
sort.Strings(names)
for _, m := range names {
why := held[m]
if was, ok := written[m]; ok && strings.HasSuffix(was.Summary, ": "+why) {
continue
}
id := HeldID(m, k.Node)
f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: HeldKind, Scope: scopeModule, ID: id, Severity: condseat.FindingSeverityWarning,
Source: HeldKind,
Subject: condseat.Subject{Scope: scopeModule, ID: id, Machine: k.Node},
Summary: fmt.Sprintf("%s is held on %s: %s", m, k.Node, why),
Headline: fmt.Sprintf("%s is held on %s", m, k.Node),
Explanation: "The node leaves this module out of its pass, keeps what runs of it and composes and applies every other module. The finding clears when the module is composed again or is no longer wanted on the node.",
At: k.now()}
if err := k.Writer.Write(ctx, f); err != nil {
k.say(fmt.Sprintf("the finding that %s is held on %s cannot be written: %v", m, k.Node, err))
continue
}
k.say(f.Summary)
}
}
func (k *HeldKeeper) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
func (k *HeldKeeper) say(s string) {
if k.Say != nil {
k.Say(s)
}
}
internal/seatwait/nodefindings.go
+574
View File
@@ -0,0 +1,574 @@
// NodeKeeper keeps the findings only the node-engine knows of itself (design 47 G A-197, ADR 0596, A-149): a have's failed
// phase (apply-failing) and its health (module-unhealthy), what it keeps of a module as it runs (kept-as-it-runs), and each
// part of its own health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed,
// directory-used-as-found, node-account-fallback), under the kind words the watchman raised them under. Each is raised and
// cleared where the engine knows the fact, through its finding writer, never read back from the have or the report by
// another part.
//
// The keeper holds the facts it was told (the haves, the last report) and, at every telling, makes the findings of this
// node those the facts say: what the writer holds of its kinds for this node and no fact says is cleared, what a fact says
// and the writer does not hold, or holds in other words, is written. Nothing runs on a timer: a bound (the apply interval,
// the minute before a failing health is said, the four hours before it is urgent) is judged at each telling, and the engine
// tells at every pass over its haves and every look at its health.
package seatwait
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// The kinds the node-engine raises of itself, the words the watchman raised them under.
const (
KindApplyFailing = "apply-failing"
KindModuleUnhealthy = "module-unhealthy"
KindKeptAsRuns = "kept-as-it-runs"
KindMachineNetwork = "machine-network"
KindNetworkRewritten = "network-rewritten"
KindNetworkUnreachable = "network-unreachable"
KindMachineUnits = "machine-units"
KindReloginNeeded = "relogin-needed"
KindUsedAsFound = "directory-used-as-found"
KindAccountFallback = "node-account-fallback"
)
// NodeKinds is every kind the keeper writes, for the finding writer's list of kinds.
var NodeKinds = []string{KindApplyFailing, KindModuleUnhealthy, KindKeptAsRuns, KindMachineNetwork, KindNetworkRewritten,
KindNetworkUnreachable, KindMachineUnits, KindReloginNeeded, KindUsedAsFound, KindAccountFallback}
// The bounds the facts are judged by.
const (
// ApplyEvery is how long a have may keep saying failed before the node says it does not apply its update.
ApplyEvery = 5 * time.Minute
// UnhealthyConfirmAfter is how long a have must keep saying its health failed before the module is said unhealthy.
UnhealthyConfirmAfter = time.Minute
// UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent.
UnhealthyUrgentAfter = 4 * time.Hour
// ControlNodeModule is the module whose have names the control node: a failing network there is urgent.
ControlNodeModule = "nats"
// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed.
PackageManagerKey = "the package manager ("
)
const (
partResolvConf = "resolv-conf"
partNames = "names"
partTunnel = "tunnel"
partBus = "bus"
stateUnhealthy = "unhealthy"
phaseFailed = nodeengineclient.PhaseFailed
)
// NodeKeeper keeps the findings of one node-engine. Tell it with Have, HaveGone and Report; every telling reconciles.
type NodeKeeper struct {
Node string
Observer string
// ObserverOf, when set, answers the observer at each write (see Waiter).
ObserverOf func() string
Writer Writer
// Ctx is the run's context for the writer's acts; nil is the background.
Ctx context.Context
// Now is the clock; nil is time.Now.
Now func() time.Time
// Say tells the node's journal; nil says nothing.
Say func(string)
mu sync.Mutex
haves map[string]nodeengine.HaveByNodeModuleValue
failedSince map[string]time.Time
sickSince map[string]time.Time
keptSince map[string]time.Time
report *nodeengine.HealthByNodeValue
byAddress map[string]string
}
// Have is a module's have as the node-engine wrote it.
func (k *NodeKeeper) Have(module string, h nodeengine.HaveByNodeModuleValue) {
k.mu.Lock()
defer k.mu.Unlock()
if k.haves == nil {
k.haves, k.failedSince, k.sickSince, k.keptSince = map[string]nodeengine.HaveByNodeModuleValue{}, map[string]time.Time{},
map[string]time.Time{}, map[string]time.Time{}
}
if h.Phase == nodeengineclient.PhaseRemoved {
k.forget(module)
k.reconcile()
return
}
k.haves[module] = h
now := k.now()
setSince(k.failedSince, module, h.Phase == phaseFailed, now)
setSince(k.sickSince, module, h.Health == nodeengineclient.HealthFailed, now)
setSince(k.keptSince, module, len(h.Kept) > 0, now)
k.reconcile()
}
// HaveGone is a module no longer held by the node.
func (k *NodeKeeper) HaveGone(module string) {
k.mu.Lock()
defer k.mu.Unlock()
k.forget(module)
k.reconcile()
}
// Report is the node's health report as it stands now (written or not), and the machines by the address of each, for the
// peers whose tunnel it does not see. Told at every look, so a bound is judged then.
func (k *NodeKeeper) Report(h nodeengine.HealthByNodeValue, byAddress map[string]string) {
k.mu.Lock()
defer k.mu.Unlock()
if h.At.IsZero() {
h.At = k.now()
}
k.report, k.byAddress = &h, byAddress
k.reconcile()
}
func (k *NodeKeeper) forget(module string) {
delete(k.haves, module)
delete(k.failedSince, module)
delete(k.sickSince, module)
delete(k.keptSince, module)
}
func setSince(m map[string]time.Time, module string, on bool, now time.Time) {
if !on {
delete(m, module)
} else if _, ok := m[module]; !ok {
m[module] = now
}
}
func (k *NodeKeeper) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
func (k *NodeKeeper) say(s string) {
if k.Say != nil {
k.Say(s)
}
}
func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") }
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
func orWords(s, or string) string {
if strings.TrimSpace(s) == "" {
return or
}
return s
}
// ownKey is the identity of a finding the keeper writes.
func ownKey(scope, id, kind string) string { return scope + "." + id + "." + kind }
// reconcile makes this node's findings of the keeper's kinds those the facts say. The caller holds the lock.
func (k *NodeKeeper) reconcile() {
ctx := k.Ctx
if ctx == nil {
ctx = context.Background()
}
want := map[string]condseat.Finding{}
add := func(f condseat.Finding) { want[ownKey(f.Scope, f.ID, f.Kind)] = f }
k.haveFindings(add)
k.reportFindings(add)
held := map[string]condseat.Finding{}
for _, f := range k.Writer.Held() {
if k.owns(f) {
held[ownKey(f.Scope, f.ID, f.Kind)] = f
}
}
for _, key := range slices.Sorted(maps.Keys(held)) {
if _, still := want[key]; still {
continue
}
f := held[key]
if err := k.Writer.Clear(ctx, f.Scope, f.ID, f.Kind, "the node-engine no longer finds it so"); err != nil {
k.say(fmt.Sprintf("the finding %s of %s cannot be cleared: %v", f.Kind, k.Node, err))
}
}
for _, key := range slices.Sorted(maps.Keys(want)) {
f := want[key]
if was, ok := held[key]; ok {
f.Since, f.At = was.Since, was.At // the same thing seen again writes the same words
if sameFinding(was, f) {
continue
}
}
if err := k.Writer.Write(ctx, f); err != nil {
k.say(fmt.Sprintf("the finding %s of %s cannot be written: %v", f.Kind, k.Node, err))
continue
}
k.say(f.Summary)
}
}
// owns says a held finding is one the keeper writes for this node.
func (k *NodeKeeper) owns(f condseat.Finding) bool {
if !slices.Contains(NodeKinds, f.Kind) {
return false
}
if f.Kind == KindNetworkUnreachable {
return strings.HasSuffix(f.ID, "."+k.Node+".unreachable")
}
return f.Subject.Machine == k.Node
}
func sameFinding(a, b condseat.Finding) bool {
if a.Summary != b.Summary || a.Severity != b.Severity || a.Headline != b.Headline || a.Needs != b.Needs ||
a.Explanation != b.Explanation || !slices.Equal(a.Subject.Also, b.Subject.Also) || len(a.Evidence) != len(b.Evidence) {
return false
}
for i := range a.Evidence {
if a.Evidence[i].Said != b.Evidence[i].Said {
return false
}
}
return true
}
// finding is a finding of this node's: observed by the engine's own module, about a machine of the scope.
//
// The id and ref are the condition's key the watchman raised it under, `<scope>.<id>.<token>` (A-14): a token other than the kind
// goes into the id as `<id>.<ref>` and is kept in ref, as the watchman's findings were written.
func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding {
since = since.UTC()
if ref != "" {
id += "." + ref
}
f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity,
Ref: ref, Source: kind, Summary: summary, At: since, Since: since,
Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}}
if said != "" {
f.Evidence = []condseat.Evidence{{At: since, Said: said}}
}
return f
}
// haveFindings are the findings the haves say: apply-failing, module-unhealthy and kept-as-it-runs.
func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) {
now := k.now()
type failure struct{ module, words string }
var failed []failure
var since time.Time
for _, m := range slices.Sorted(maps.Keys(k.failedSince)) {
failed = append(failed, failure{m, firstLine(orWords(k.haves[m].Detail, "no reason given"))})
if at := k.failedSince[m]; since.IsZero() || at.Before(since) {
since = at
}
}
if len(failed) > 0 && now.Sub(since) > ApplyEvery {
var names, words []string
tool := ""
for _, f := range failed {
names = append(names, f.module)
words = append(words, f.module+": "+f.words)
w := strings.TrimPrefix(f.words, "refused: ")
if i := strings.Index(w, PackageManagerKey); i >= 0 && tool == "" {
tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")")
}
}
f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; "))
f.Headline = k.Node + " is not applying its update"
if tool != "" {
f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)",
k.Node, tool, stampMin(since), ApplyEvery)
f.Headline = k.Node + " cannot install packages"
} else {
f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", k.Node, strings.Join(names, ", "), stampMin(since), ApplyEvery)
f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", k.Node)
}
add(f)
}
for _, m := range slices.Sorted(maps.Keys(k.sickSince)) {
at := k.sickSince[m]
if now.Sub(at) < UnhealthyConfirmAfter {
continue
}
severity := condseat.FindingSeverityWarning
if now.Sub(at) >= UnhealthyUrgentAfter {
severity = condseat.FindingSeverityUrgent
}
h := k.haves[m]
f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at,
fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed")))
f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node)
add(f)
}
for _, m := range slices.Sorted(maps.Keys(k.keptSince)) {
words := strings.Join(k.haves[m].Kept, "; ")
f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m],
fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words))
f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node)
f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)"
add(f)
}
}
func stampMin(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") }
// control says the node runs the module the control node is named by.
func (k *NodeKeeper) control() bool {
h, ok := k.haves[ControlNodeModule]
return ok && h.Phase != nodeengineclient.PhaseRemoved
}
// reportFindings are the findings the node's health report says.
func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) {
h := k.report
if h == nil {
return
}
node := k.Node
var own []nodeengine.NetworkPart
var rewritten *nodeengine.NetworkPart
var parts []nodeengine.NetworkPart
if h.Network != nil {
for _, p := range h.Network.Parts {
if p.State == stateUnhealthy {
parts = append(parts, p)
}
}
}
toward := map[string][]nodeengine.NetworkPart{}
for i, p := range parts {
switch {
case p.Part == partResolvConf:
rewritten = &parts[i]
case len(p.Targets) > 0:
for _, x := range p.Targets {
toward[x] = append(toward[x], p)
}
default:
own = append(own, p)
}
}
if rewritten != nil {
add(k.rewritten(*rewritten, parts))
kept := own[:0]
for _, p := range own {
if p.Part != partNames {
kept = append(kept, p)
}
}
own = kept
}
if len(own) > 0 {
var words, said []string
severity := condseat.FindingSeverityWarning
for _, p := range own {
words = append(words, p.Reason)
said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said))
if p.Part == partBus {
severity = condseat.FindingSeverityUrgent
}
}
if k.control() {
severity = condseat.FindingSeverityUrgent
}
add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since,
fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | ")))
}
// A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would.
for _, p := range h.Peers {
if p.Seen {
continue
}
for _, a := range p.Addresses {
addr, _, _ := strings.Cut(a, "/")
if x := k.byAddress[addr]; x != "" && x != node {
toward[x] = append(toward[x], nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, Since: h.At,
Said: "no handshake with " + x + " within five minutes", Targets: []string{x}})
}
}
}
for _, x := range slices.Sorted(maps.Keys(toward)) {
var what, said []string
since := toward[x][0].Since
severity := condseat.FindingSeverityWarning
for _, p := range toward[x] {
wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[p.Part]
if wd == "" {
wd = p.Part
}
if !slices.Contains(what, wd) {
what = append(what, wd)
}
said = append(said, fmt.Sprintf("%s: %s: %s", node, p.Part, p.Said))
if p.Since.Before(since) {
since = p.Since
}
if p.Part == partBus {
severity = condseat.FindingSeverityUrgent
}
}
f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since,
fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | "))
f.Subject.Also = []string{node}
add(f)
}
if f, ok := k.units(h.Units); ok {
add(f)
}
for _, f := range k.relogin(h.Relogin) {
add(f)
}
for _, f := range k.found(h.Found) {
add(f)
}
if f := h.Fallback; f != nil && f.Node == node {
said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why)
g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said)
g.Headline = fmt.Sprintf("%s fell back to its old bus login", node)
g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node)
add(g)
}
}
func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.NetworkPart) condseat.Finding {
m := k.Node
writer := ""
if p.Writer != "" {
writer = " (" + p.Writer + ")"
}
cost := "the names still resolve through what it wrote"
said := []string{p.Part + ": " + p.Said}
for _, q := range all {
if q.Part == partNames {
cost = strings.TrimSuffix(q.Reason, ".")
said = append(said, q.Part+": "+q.Said)
}
}
id := m
summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+
"back at its next reconcile, or that program gives it back", m, writer, cost)
if p.Owner != "" {
id = m + "." + p.Owner
summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+
"writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
}
severity := condseat.FindingSeverityWarning
if k.control() {
severity = condseat.FindingSeverityUrgent
}
return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary,
fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | ")))
}
func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) {
if u == nil || len(u.Failed) == 0 {
return condseat.Finding{}, false
}
node := k.Node
var names, said []string
since := u.Failed[0].Since
for _, f := range u.Failed {
where := ""
if f.Scope == "user" {
where = " (the account's own manager)"
}
names = append(names, f.Unit+where)
scope, result := f.Scope, f.Result
if scope == "" {
scope = "system"
}
if result == "" {
result = "no reason said"
}
line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result)
if f.Load != "" && f.Load != "loaded" {
line += ", its unit " + f.Load
}
if f.Resource != "" {
line += ", named by the mesh's own " + f.Resource
}
said = append(said, line+", since "+stampSec(f.Since))
if f.Since.Before(since) {
since = f.Since
}
}
failed := "1 failed service"
if len(u.Failed) > 1 {
failed = fmt.Sprintf("%d failed services", len(u.Failed))
}
f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since,
fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+
"mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")),
strings.Join(said, "; "))
f.Headline = strings.ToUpper(failed[:1]) + failed[1:] + " on " + node
f.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
return f, true
}
func orModule(m string) string {
if m == "" {
return "_"
}
return m
}
func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Finding {
by := map[string][]nodeengine.NodeAccount{}
for _, a := range accounts {
by[a.Module] = append(by[a.Module], a)
}
var out []condseat.Finding
for _, module := range slices.Sorted(maps.Keys(by)) {
var names []string
since := by[module][0].Since
for _, a := range by[module] {
names = append(names, a.Account)
if a.Since.Before(since) {
since = a.Since
}
}
sort.Strings(names)
said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+
"before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", "))
out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said))
}
return out
}
func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding {
by := map[string][]nodeengine.NodeDirectory{}
for _, d := range dirs {
by[d.Module] = append(by[d.Module], d)
}
var out []condseat.Finding
for _, module := range slices.Sorted(maps.Keys(by)) {
var ids, why []string
since := by[module][0].Since
for _, d := range by[module] {
ids = append(ids, d.Resource)
why = append(why, d.Reason)
if d.Since.Before(since) {
since = d.Since
}
}
m := orModule(module)
said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; "))
f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said)
f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node)
f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+
"and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m)
f.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type."
out = append(out, f)
}
return out
}
internal/seatwait/nodefindings_test.go
+104
View File
@@ -0,0 +1,104 @@
package seatwait
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// Design 47 G A-197: what only the node-engine knows of itself it raises as its own findings, under the kind words the watchman
// raised them under, and clears where it knows the fact no longer holds.
func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) {
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
type step struct {
do func(k *NodeKeeper, advance func(time.Duration))
want []string // "scope.id.kind" of what stands after it, sorted
}
failed := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseFailed, Detail: "refused: the package manager (pacman) does not answer"}
sick := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "port closed"}
report := func(h nodeengine.HealthByNodeValue) func(*NodeKeeper, func(time.Duration)) {
return func(k *NodeKeeper, _ func(time.Duration)) { k.Report(h, map[string]string{"10.0.0.2": "g14"}) }
}
for name, steps := range map[string][]step{
"a have failed past the apply interval is apply-failing, cleared by the clean apply": {
{func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", failed) }, nil},
{func(k *NodeKeeper, adv func(time.Duration)) { adv(ApplyEvery + time.Second); k.Have("b", failed) }, []string{"machine.ace.apply-failing"}},
{func(k *NodeKeeper, _ func(time.Duration)) {
k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning})
}, nil},
},
"a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": {
{func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil},
{func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.unhealthy.module-unhealthy"}},
{func(k *NodeKeeper, _ func(time.Duration)) {
k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved})
}, nil},
},
"what a module keeps as it runs is kept-as-it-runs until its have says none": {
{func(k *NodeKeeper, _ func(time.Duration)) {
k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}})
}, []string{"module.b.ace.kept.kept-as-it-runs"}},
{func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil},
},
"each part of the report is its own finding and each ends with the report": {
{report(nodeengine.HealthByNodeValue{
Network: &nodeengine.NetworkHealth{Since: at, Parts: []nodeengine.NetworkPart{
{Part: "bus", State: "unhealthy", Reason: "no bus", Since: at},
{Part: "tunnel", State: "unhealthy", Since: at, Targets: []string{"g14"}},
{Part: "resolv-conf", State: "unhealthy", Owner: "dnsmasq", Since: at}}},
Units: &nodeengine.UnitsHealth{Failed: []nodeengine.FailedUnit{{Unit: "x.service", Since: at}}},
Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}},
Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}},
Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}),
[]string{"machine.ace.dnsmasq.rewritten.network-rewritten", "machine.ace.network.machine-network", "machine.ace.node-account-fallback",
"machine.ace.units.machine-units", "machine.g14.ace.unreachable.network-unreachable", "module.m.ace.directory-used-as-found",
"module.m.ace.relogin-needed"}},
{report(nodeengine.HealthByNodeValue{}), nil},
},
"a tunnel peer not seen is the machine unreachable": {
{report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}),
[]string{"machine.g14.ace.unreachable.network-unreachable"}},
{report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil},
},
} {
t.Run(name, func(t *testing.T) {
now := at
w := &fakeFindings{}
// Another node's finding and an unrelated kind stand in the state and are never touched.
w.held = []condseat.Finding{{Scope: "machine", ID: "g14", Kind: KindMachineUnits, Subject: condseat.Subject{Machine: "g14"}},
{Scope: "machine", ID: "ace", Kind: HeldKind, Subject: condseat.Subject{Machine: "ace"}}}
k := &NodeKeeper{Node: "ace", Observer: "mesh-host", Writer: w, Now: func() time.Time { return now }}
for i, s := range steps {
s.do(k, func(d time.Duration) { now = now.Add(d) })
var got []string
for _, f := range w.held {
if f.Observer == "mesh-host" {
got = append(got, f.Scope+"."+f.ID+"."+f.Kind)
}
}
if strings.Join(sorted(got), ",") != strings.Join(s.want, ",") {
t.Fatalf("step %d: stands %v, want %v", i, sorted(got), s.want)
}
}
if len(w.held) != 2 {
t.Fatalf("what is not the keeper's was touched: %+v", w.held)
}
})
}
}
func sorted(in []string) []string {
out := append([]string(nil), in...)
for i := range out {
for j := i + 1; j < len(out); j++ {
if out[j] < out[i] {
out[i], out[j] = out[j], out[i]
}
}
}
return out
}
internal/seatwait/seatwait.go
+68 -27
View File
@@ -8,12 +8,13 @@
package seatwait
import (
"context"
"fmt"
"sync"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
)
// Bound is how long the reader may wait before the node says it (ADR 0636 §1).
@@ -36,21 +37,44 @@ type Clock interface {
AfterFunc(d time.Duration, f func()) Timer
}
// Findings is the node's own state `findings`: the writer, and the read of whether a finding stands there. The bus state
// is the one truth of whether the finding exists (ADR 0636 §1, issue 1966): the Waiter keeps no copy, so a new process
// of the node-engine clears what an earlier one wrote.
type Findings interface {
views.FindingWriter
Has(f views.Finding) (bool, error)
// Writer is the node-engine's finding writer as the keepers use it: mesh-sdk's held writer over the node's state `findings`
// (views.HeldWriter, design 47 G A-29, A-38, A-40), which holds what stands there from its start, so the Waiter keeps no
// copy and a new process of the node-engine clears what an earlier one wrote (ADR 0636 §1, issue 1966).
type Writer interface {
Write(ctx context.Context, f condseat.Finding) error
Clear(ctx context.Context, scope, id, kind, why string) error
Held() []condseat.Finding
}
const (
scopeMachine = "machine"
scopeModule = "module"
)
// stands is whether the writer holds the finding of that scope, id and kind.
func stands(w Writer, scope, id, kind string) bool {
for _, f := range w.Held() {
if f.Scope == scope && f.ID == id && f.Kind == kind {
return true
}
}
return false
}
// Waiter writes and clears the node's finding.
type Waiter struct {
Node string
Observer string
Reader Reader
Findings Findings
Clock Clock
// ObserverOf, when set, answers the observer at each write: the node-engine's own module, which a first start learns
// from its first composing (design 47 G §527, §634).
ObserverOf func() string
Reader Reader
Writer Writer
Clock Clock
// Bound is how long the reader may wait; zero is the default Bound (the manifest's config `view-waits-minutes`, §633).
Bound time.Duration
// Ctx is the run's context for the writer's acts; nil is the background.
Ctx context.Context
// Say tells the node's journal; nil says nothing.
Say func(string)
@@ -59,13 +83,20 @@ type Waiter struct {
since time.Time
}
// key is the finding this node's Waiter writes: Kind and ID name it, the rest is its text.
func (w *Waiter) key() views.Finding {
return views.Finding{Observer: w.Observer, Kind: Kind, ID: w.Node}
func (w *Waiter) ctx() context.Context {
if w.Ctx != nil {
return w.Ctx
}
return context.Background()
}
func (w *Waiter) stands() bool { return stands(w.Writer, scopeMachine, w.Node, Kind) }
func waiting(st seatlist.Status) bool { return !st.Heard || len(st.Waiting) > 0 }
// Waits is whether a reader of that status holds only the definitions it started with, or waits for some.
func Waits(st seatlist.Status) bool { return waiting(st) }
// Changed is told when the reader's held set or its status moved: it starts the bound when the reader waits and has no
// bound running, stops it and clears the finding when the reader has read the whole list.
func (w *Waiter) Changed() {
@@ -78,13 +109,8 @@ func (w *Waiter) Changed() {
w.timer = nil
}
w.since = time.Time{}
stands, err := w.Findings.Has(w.key())
if err != nil {
w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be read: %v", err))
return
}
if stands {
if err := w.Findings.Delete(w.key()); err != nil {
if w.stands() {
if err := w.Writer.Clear(w.ctx(), scopeMachine, w.Node, Kind, "the node-engine has read the whole seat list"); err != nil {
w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be cleared: %v", err))
}
}
@@ -93,11 +119,11 @@ func (w *Waiter) Changed() {
if w.timer != nil {
return
}
if stands, err := w.Findings.Has(w.key()); err == nil && stands {
if w.stands() {
return // an earlier process wrote it and the list is still unread: it stands until the first full read
}
w.since = w.Clock.Now()
w.timer = w.Clock.AfterFunc(Bound, w.expired)
w.timer = w.Clock.AfterFunc(w.bound(), w.expired)
}
// Waiting says whether the bound is running or the finding stands, and since when.
@@ -107,8 +133,7 @@ func (w *Waiter) Waiting() (bool, time.Time) {
if w.timer != nil {
return true, w.since
}
stands, err := w.Findings.Has(w.key())
return err == nil && stands, w.since
return w.stands(), w.since
}
func (w *Waiter) expired() {
@@ -120,15 +145,16 @@ func (w *Waiter) expired() {
w.since = time.Time{}
return
}
if stands, err := w.Findings.Has(w.key()); err == nil && stands {
if w.stands() {
return
}
f := views.Finding{Observer: w.Observer, Kind: Kind, ID: w.Node,
f := condseat.Finding{Observer: observerOf(w.Observer, w.ObserverOf), Kind: Kind, Scope: scopeMachine, ID: w.Node, Severity: condseat.FindingSeverityWarning,
Source: Kind, Subject: condseat.Subject{Scope: scopeMachine, ID: w.Node, Machine: w.Node},
Summary: fmt.Sprintf("%s has read no complete seat list since %s: %s", w.Node, w.since.UTC().Format(time.RFC3339), st.Health()),
Headline: fmt.Sprintf("%s waits for the seat list", w.Node),
Explanation: "The node holds only the definitions it starts with, so it keeps running what it runs and starts, composes and grants nothing new. It reads the list again at the next change the seat mesh-catalog says, or when asked to.",
At: w.Clock.Now()}
if err := w.Findings.Put(f); err != nil {
if err := w.Writer.Write(w.ctx(), f); err != nil {
w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be written: %v", err))
return
}
@@ -140,3 +166,18 @@ func (w *Waiter) say(s string) {
w.Say(s)
}
}
// observerOf is of() when it is given, else fixed.
func observerOf(fixed string, of func() string) string {
if of != nil {
return of()
}
return fixed
}
func (w *Waiter) bound() time.Duration {
if w.Bound > 0 {
return w.Bound
}
return Bound
}
internal/seatwait/seatwait_test.go
+110 -25
View File
@@ -1,12 +1,15 @@
package seatwait
import (
"context"
"errors"
"strings"
"testing"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
)
type fakeReader struct{ st seatlist.Status }
@@ -45,25 +48,37 @@ func (c *fakeClock) advance(d time.Duration) {
}
}
// fakeFindings is a held writer: what it holds is what stands in the state, seeded by the test for an earlier process's.
type fakeFindings struct {
put, deleted []views.Finding
err error
standing bool // the bus state holds the finding, written by this process or an earlier one
put []condseat.Finding
cleared []condseat.Finding // as held when cleared
why []string
err error
held []condseat.Finding
}
func (f *fakeFindings) Has(views.Finding) (bool, error) { return f.standing, nil }
func (f *fakeFindings) Held() []condseat.Finding { return append([]condseat.Finding(nil), f.held...) }
func (f *fakeFindings) Put(x views.Finding) error {
func (f *fakeFindings) Write(_ context.Context, x condseat.Finding) error {
if f.err != nil {
return f.err
}
f.put = append(f.put, x)
f.standing = true
f.held = append(f.held, x)
return nil
}
func (f *fakeFindings) Delete(x views.Finding) error {
f.deleted = append(f.deleted, x)
f.standing = false
func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, why string) error {
for i, h := range f.held {
if h.Scope == scope && h.ID == id && h.Kind == kind {
f.held = append(f.held[:i:i], f.held[i+1:]...)
f.cleared = append(f.cleared, h)
f.why = append(f.why, why)
return nil
}
}
f.cleared = append(f.cleared, condseat.Finding{Scope: scope, ID: id, Kind: kind}) // cleared of nothing, said all the same
f.why = append(f.why, why)
return nil
}
@@ -73,7 +88,7 @@ func caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead(t
clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)}
findings := &fakeFindings{}
var said []string
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock, Say: func(s string) { said = append(said, s) }}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Say: func(s string) { said = append(said, s) }}
w.Changed()
clock.advance(Bound - time.Second)
@@ -92,8 +107,8 @@ func caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead(t
reader.st = seatlist.Status{Heard: true}
w.Changed()
if len(findings.deleted) != 1 || findings.deleted[0].ID != "ace" {
t.Fatalf("the first full read left the finding: deleted %+v", findings.deleted)
if len(findings.cleared) != 1 || findings.cleared[0].ID != "ace" {
t.Fatalf("the first full read left the finding: deleted %+v", findings.cleared)
}
if on, _ := w.Waiting(); on {
t.Error("the waiter still waits after the full read")
@@ -105,14 +120,14 @@ func caseAReadWithinTheBoundWritesNothing(t *testing.T) {
reader := &fakeReader{st: seatlist.Status{Heard: false}}
clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)}
findings := &fakeFindings{}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock}
w.Changed()
clock.advance(time.Minute)
reader.st = seatlist.Status{Heard: true}
w.Changed()
clock.advance(time.Hour)
if len(findings.put) != 0 || len(findings.deleted) != 0 {
t.Fatalf("a list read in time wrote %+v and cleared %+v", findings.put, findings.deleted)
if len(findings.put) != 0 || len(findings.cleared) != 0 {
t.Fatalf("a list read in time wrote %+v and cleared %+v", findings.put, findings.cleared)
}
// The seat answered the list already held: no change was said, so the timer is the one to look again.
@@ -131,7 +146,7 @@ func caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)}
findings := &fakeFindings{err: errors.New("bus down")}
var said []string
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock, Say: func(s string) { said = append(said, s) }}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Say: func(s string) { said = append(said, s) }}
w.Changed()
clock.advance(Bound)
if len(said) != 1 || len(findings.put) != 0 {
@@ -150,8 +165,8 @@ func caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain(t *testing.T) {
func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) {
reader := &fakeReader{st: seatlist.Status{Heard: false}}
clock := &fakeClock{now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
findings := &fakeFindings{standing: true}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock}
findings := &fakeFindings{held: []condseat.Finding{{Scope: "machine", ID: "ace", Kind: Kind}}}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock}
w.Changed()
clock.advance(Bound)
if len(findings.put) != 0 || len(clock.timers) != 0 {
@@ -162,18 +177,88 @@ func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) {
}
reader.st = seatlist.Status{Heard: true}
w.Changed()
if len(findings.deleted) != 1 || findings.deleted[0].ID != "ace" || len(findings.put) != 0 {
t.Fatalf("the first full read left the earlier process's finding: deleted %+v, put %+v", findings.deleted, findings.put)
if len(findings.cleared) != 1 || findings.cleared[0].ID != "ace" || len(findings.put) != 0 {
t.Fatalf("the first full read left the earlier process's finding: deleted %+v, put %+v", findings.cleared, findings.put)
}
}
// Design 47, test 55: a module held for its claim is a finding that names the node, the module and why, written once,
// and cleared when the module is composed again or no longer wanted, also one an earlier run of the engine left.
func caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted(t *testing.T) {
// the writer holds what an earlier run of the engine left (seeded at its Start), and what another node's keeper wrote
findings := &fakeFindings{held: []condseat.Finding{
{Scope: "module", ID: HeldID("gone", "ace"), Kind: HeldKind},
{Scope: "module", ID: HeldID("elsewhere", "g14"), Kind: HeldKind}}}
k := &HeldKeeper{Node: "ace", Observer: "node-engine", Writer: findings}
k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"})
if len(findings.put) != 2 || findings.put[0].ID != "mesh-grants.ace" || len(findings.cleared) != 1 || findings.cleared[0].ID != "gone.ace" {
t.Fatalf("wrote %v, cleared %v", findings.put, findings.cleared)
}
// the finding is of the module on its machine, in the contract's layout (design 47, test 55)
if f := findings.put[0]; f.Scope != "module" || f.Subject.Scope != "module" || f.Subject.Machine != "ace" || f.Subject.ID != "mesh-grants.ace" ||
f.Severity != condseat.FindingSeverityWarning || f.Source != HeldKind || f.Observer != "node-engine" {
t.Fatalf("a held module's finding is %+v", f)
}
if got := findings.held[0]; got.ID != "elsewhere.g14" {
t.Fatalf("another node's finding was touched: %+v", findings.held)
}
k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"})
if len(findings.put) != 2 {
t.Fatalf("a module held for the same words was written again: %v", findings.put)
}
k.Reconcile(map[string]string{"mesh-grants": "its seat is refused"})
if len(findings.cleared) != 2 || findings.cleared[1].ID != "other.ace" {
t.Fatalf("a module composed again kept its finding: %v", findings.cleared)
}
k.Reconcile(nil)
if len(findings.cleared) != 3 {
t.Fatalf("a module no longer wanted kept its finding: %v", findings.cleared)
}
}
// Design 47 G §614, A-117: the finding entry-not-found is of scope module with the fingerprint, escaped, as its id, and cleared by it.
func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) {
w := &fakeFindings{}
fp := "src2:" + strings.Repeat("a", 64)
now := func() time.Time { return time.Unix(1, 0) }
if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, true, now); err != nil {
t.Fatal(err)
}
if len(w.put) != 1 || w.put[0].Kind != EntryNotFoundKind || w.put[0].Scope != "module" || w.put[0].ID != keys.EscapeName(fp) || strings.Contains(w.put[0].ID, ":") {
t.Fatalf("written %+v", w.put)
}
if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, false, now); err != nil || len(w.cleared) != 1 || w.cleared[0].ID != w.put[0].ID {
t.Fatalf("cleared %+v, %v", w.cleared, err)
}
}
// ADR 0636 §1, the node's finding of a seat list unread.
// §633: the bound is the manifest's `view-waits-minutes` when one is given: a finding at two minutes, none before.
func caseAGivenBoundReplacesTheFiveMinutes(t *testing.T) {
reader := &fakeReader{st: seatlist.Status{Heard: false, LastError: "no answer"}}
clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)}
findings := &fakeFindings{}
w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Bound: 2 * time.Minute}
w.Changed()
clock.advance(2*time.Minute - time.Second)
if len(findings.put) != 0 {
t.Fatalf("a finding was written before the given bound: %+v", findings.put)
}
clock.advance(time.Second)
if len(findings.put) != 1 {
t.Fatalf("no finding at the given bound: %+v", findings.put)
}
}
func TestTheSeatListWait(t *testing.T) {
for name, run := range map[string]func(*testing.T){
"five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead,
"a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing,
"a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain,
"a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead,
"five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead,
"a given bound replaces the five minutes": caseAGivenBoundReplacesTheFiveMinutes,
"a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing,
"a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain,
"a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead,
"an entry not found is a finding of the fingerprint": caseAnEntryNotFoundIsAFindingOfTheFingerprint,
"a held module is a finding cleared when it is composed or no longer wanted": caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted,
} {
t.Run(name, run)
}
internal/selfcompose/candidate.go
+8 -5
View File
@@ -6,9 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/buckets"
engcomp "github.com/novox/mesh-host/internal/compositions"
"maps"
@@ -322,12 +322,15 @@ func (c *Composer) scratch() *Composer {
out.values = maps.Clone(c.values)
out.manifests = maps.Clone(c.manifests)
out.archives = maps.Clone(c.archives)
out.builds = maps.Clone(c.builds)
// The entries it holds and the wants that name them give a build (buildOf), as the builder's builds once did.
out.keptEnt = maps.Clone(c.keptEnt)
out.wants = maps.Clone(c.wants)
out.sealedOwn = maps.Clone(c.sealedOwn)
out.ownHanded = maps.Clone(c.ownHanded)
out.certified = c.certified
out.assignmentKeys = c.assignmentKeys
out.origin = c
out.forCandidates = c.forCandidates
return out
}
@@ -461,7 +464,7 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed
c.mu.Lock()
c.wants = wants
c.mu.Unlock()
scope := frombus.Scope{Node: node, KeptAt: kept, InOwn: inOwn, Running: running, SameValue: c.sameValue}
scope := frombus.Scope{Node: node, KeptAt: kept, InOwn: inOwn, Running: running, SameValue: c.sameValue, Heard: c.heardNow()}
if input != nil {
input(scope, fps)
}
internal/selfcompose/candidate_test.go
+16 -6
View File
@@ -4,9 +4,10 @@ import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"strings"
@@ -78,6 +79,15 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
env.ComposedState = state
var applied atomic.Int64
env.Refused = func(string, map[string]string) { applied.Add(1) }
// What the node's want names is read from the catalog half's entry (design 47 G §421, A-308): the shadow watches no modules.
var manifest map[string]any
if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil {
t.Fatal(err)
}
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", ManifestDigest: digest}
env.Entry = EntryCall(func(_ context.Context, _, _ string, _ any) (json.RawMessage, error) {
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
var handed atomic.Int64
s := NewShadow(env, func(context.Context, map[string]nodeengine.CandidateAnswer) { handed.Add(1) })
ctx, cancel := context.WithCancel(context.Background())
@@ -89,7 +99,7 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"},
Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}})
have, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"})
want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true})
deadline := time.Now().Add(5 * time.Second)
watching := func() int { b.mu.Lock(); defer b.mu.Unlock(); return len(b.watches) }
for watching() < 6 && time.Now().Before(deadline) {
@@ -97,8 +107,8 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
}
b.putAssigned(t, place)
b.put(t, buckets.Have, "n1.node", have)
b.put(t, buckets.Modules, "lemurs", mod)
for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket} {
b.put(t, buckets.Wants, "want.n1.lemurs", want)
for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Wants, ResolvedBucket} {
b.send(bucket, agent.Update{Caught: true})
}
for !s.ready.Load() && time.Now().Before(deadline) {
internal/selfcompose/compose_while_passing_test.go
+2 -3
View File
@@ -10,8 +10,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
@@ -48,7 +47,7 @@ func composeAnsweredWhileAPassRuns(t *testing.T) {
b.putAssigned(t, place)
b.put(t, buckets.Have, "n1.node", have)
b.put(t, buckets.Wants, "want.n1.node", nodeWant)
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} {
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} {
b.send(bucket, agent.Update{Caught: true})
}
go func() {
internal/selfcompose/composedstate.go
+68 -22
View File
@@ -89,31 +89,20 @@ func (c *Composer) publishComposed(ctx context.Context) {
// is left out. A module whose newest build the builder holds failed is also returned in failed, by module, with the
// indexer's fingerprint and the build's error: it is published as a refusal, not composed.
func (c *Composer) newestFPs(ctx context.Context, at *published) (out map[string]string, failed map[string][2]string) {
out = map[string]string{}
// The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node
// follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. A placed module's newest
// build that ended is the last trunk end the builder said of it (NoteBuildEnded): a failed one is refused (A-293).
failed = map[string][2]string{}
r := reader{c: c}
for _, module := range at.placed {
c.mu.Lock()
fp2 := c.modules[module].Fp2
c.mu.Unlock()
if fp2 != "" {
if b, err := r.Build(ctx, module, fp2); err == nil && b != nil {
switch b.State {
case builds.BuildDone:
out[module] = fp2
continue
case builds.BuildFailed:
failed[module] = [2]string{fp2, b.Error}
}
c.mu.Lock()
if c.endsRead {
for _, module := range at.placed {
if e, ok := c.ends[module]; ok && e.Outcome == builds.BuildFailed {
failed[module] = [2]string{e.Fingerprint, e.Error}
}
}
if fp, ok := at.fps[module]; ok {
out[module] = fp
} else if fp, ok := at.ran[module]; ok {
out[module] = fp
}
}
return out, failed
c.mu.Unlock()
return maps.Clone(at.fps), failed
}
// writeComposed works out the state's keys from at and writes those that differ, then deletes what is past keeping.
@@ -146,8 +135,12 @@ func (c *Composer) writeComposed(ctx context.Context, at *published) (wrote, del
}
c.pubKept[key] = ComposedEntry{Value: values[key], Revision: rev}
wrote++
c.saidComposed(key, values[key])
}
for _, key := range c.pastKeeping() {
for _, key := range append(c.pastKeeping(), c.endedRefusals(values)...) {
if _, kept := c.pubKept[key]; !kept {
continue // past keeping and ended both
}
if err := c.env.ComposedState.Delete(ctx, key); err != nil {
return wrote, deleted, fmt.Errorf("%s cannot be deleted: %w", key, err)
}
@@ -265,6 +258,38 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin
return out, nil
}
// endedRefusals is, of the keys the state holds, each refusal a failed end of a module wrote (newestFPs) once that module's
// last trunk end is done, at any fingerprint (design 47 G A-293): the state never says refused for a done build. A key this
// pass writes (values), or one a want names, is not one. Under pubWrite.
func (c *Composer) endedRefusals(values map[string][]byte) []string {
c.mu.Lock()
defer c.mu.Unlock()
if !c.endsRead {
return nil
}
var out []string
for module, last := range c.ends {
if last.Outcome != builds.BuildDone {
continue
}
for fp := range c.endFPs[module] {
key, err := engcomp.Key(c.env.Node, module, fp)
if err != nil || values[key] != nil || c.wants[module].Fp == fp {
continue
}
kept, ok := c.pubKept[key]
if !ok {
continue
}
if comp, err := engcomp.Decode(kept.Value); err == nil && comp.Refused != "" {
out = append(out, key)
}
}
}
sort.Strings(out)
return out
}
// composedKeyOf is a composition's key in the state.
func composedKeyOf(comp engcomp.Composed) (string, error) {
if comp.Module == "" {
@@ -341,3 +366,24 @@ func ranOf(running []engcomp.Composed) map[string]string {
}
return out
}
// saidComposed says the event composed for a key just written that holds a module's composition at a fingerprint and was
// not refused: the node's own composition and a refusal say nothing.
func (c *Composer) saidComposed(key string, value []byte) {
if c.env.SayComposed == nil {
return
}
_, module, fp, isNode, err := engcomp.ParseKey(key)
if err != nil || isNode {
return
}
comp, err := engcomp.Decode(value)
if err != nil || comp.Refused != "" {
return
}
digest, err := engcomp.DigestOf(comp)
if err != nil {
return
}
c.env.SayComposed(module, fp, digest)
}
internal/selfcompose/composedstate_test.go
+111 -63
View File
@@ -2,16 +2,19 @@ package selfcompose
import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
engcomp "github.com/novox/mesh-host/internal/compositions"
"maps"
"slices"
"strings"
"sync"
@@ -60,13 +63,31 @@ func (f *fakeComposed) Delete(_ context.Context, key string) error {
func (f *fakeComposed) written() []string {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.puts)
var out []string
for _, key := range f.puts {
// The entries and wants the node keeps are its composition state's too (§683), not compositions.
if !strings.HasPrefix(key, keptEntryPrefix) && !strings.HasPrefix(key, keptWantPrefix) {
out = append(out, key)
}
}
return out
}
// keep holds in the state, as an earlier run of the node left them, the entry of fp and the want of module at it (§683): the
// composer then composes at its start from what it kept, and asks the catalog half of nothing first.
func (f *fakeComposed) keep(t *testing.T, half catalogHalf, module, fp string) {
t.Helper()
entry, _ := json.Marshal(meshcatalog.EntryAnswer{Entry: half[fp]})
want, _ := json.Marshal(keptWant{V: 1, Since: time.Now(), Want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}})
f.mu.Lock()
defer f.mu.Unlock()
f.kv[entryKey(fp)] = ComposedEntry{Value: entry, Revision: 1}
f.kv[wantStateKey(module)] = ComposedEntry{Value: want, Revision: 1}
}
// novox/hq ADR 0593 §6: the node publishes each composition it makes in its composition state, a record never applied:
// the wanted module and the node's own at the first pass; nothing again for a pass that changes nothing; a done build
// of a module placed here that no want names, at its trunk's build, written once as its own key (before any want moves);
// the same build landing again writes nothing. And a module's keys are kept while among the five it wrote last or named
// the wanted module and the node's own at the first pass, from the entry and want it kept; nothing again for a pass that
// changes nothing; a failed build of a placed module said refused by its build-ended event. And a module's keys are kept while among the five it wrote last or named
// by a want, and deleted otherwise.
func TestTheNodePublishesItsCompositions(t *testing.T) {
t.Run("the verb compose is answered while a pass runs", composeAnsweredWhileAPassRuns)
@@ -87,6 +108,20 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
state := &fakeComposed{kv: map[string]ComposedEntry{}}
env := b.env()
env.ComposedState = state
ofp0 := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
half := catalogHalf{}
half.add(t, "lemurs", fp, lemurs)
half.add(t, "oldie", ofp0, oldie)
env.Entry = half.call()
state.keep(t, half, "lemurs", fp)
// Design 47, *What each step says*: each module composition written says composed, with its fingerprint and digest.
var saidMu sync.Mutex
var saidComposed []string
env.SayComposed = func(module, fingerprint, digest string) {
saidMu.Lock()
defer saidMu.Unlock()
saidComposed = append(saidComposed, module+" "+fingerprint+" "+digest)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := New(env)
@@ -113,13 +148,12 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
}
b.put(t, buckets.Wants, "want.n1.lemurs", want)
b.put(t, buckets.Wants, "want.n1.node", nodeWant)
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} {
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} {
b.send(bucket, agent.Update{Caught: true})
}
for u := next(t, out); !u.Caught; u = next(t, out) {
}
lemursKey, _ := engcomp.Key("n1", "lemurs", fp)
oldieKey, _ := engcomp.Key("n1", "oldie", ofp)
waitFor := func(what string, ok func([]string) bool) []string {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
@@ -145,6 +179,31 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
t.Errorf("the published composition of lemurs: %+v, %v", comp, err)
}
}
if comp, err := engcomp.Decode(state.kv[lemursKey].Value); err == nil {
dig, _ := engcomp.DigestOf(comp)
saidMu.Lock()
if len(saidComposed) != 1 || saidComposed[0] != "lemurs "+fp+" "+dig || dig == "" {
t.Errorf("composed said %v, want once for lemurs at %s with its digest %q (the node's own says none)", saidComposed, fp, dig)
}
saidMu.Unlock()
}
// Design 47 G §199, §556 (test 56): what the nodes say they run is acted on once the first read of the stream ended; the
// newest by `at` stands, an older one and the same again change nothing.
at := time.Now()
c.NoteRunning(nodeengine.Running{Node: "n2", Module: "lemurs", Fingerprint: "a", Manifest: "sha256:a", At: at})
if got := c.heardNow(); got != nil {
t.Errorf("heard %v before the first read ended", got)
}
c.RunningRead()
c.NoteRunning(nodeengine.Running{Node: "n2", Module: "lemurs", Fingerprint: "old", At: at.Add(-time.Hour)})
c.NoteRunning(nodeengine.Running{Node: "n3", Module: "lemurs", Stopped: true, At: at})
if got := c.heardNow(); len(got) != 2 || got["n2"]["lemurs"].Fingerprint != "a" || !got["n3"]["lemurs"].Stopped {
t.Errorf("heard %v, want n2's lemurs at a (not the older) and n3's stopped", got)
}
c.RunningLost()
if got := c.heardNow(); got != nil {
t.Errorf("heard %v with the stream lost", got)
}
// A pass that changes nothing writes nothing.
setFor(t, &askedWithin, 0)
if _, err := c.ComposeNow(ctx); err != nil {
@@ -154,60 +213,6 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
if got := state.written(); len(got) != 2 {
t.Errorf("a pass that changed nothing wrote %v", got[2:])
}
// oldie's trunk build is done: its key at that build is written once, though no want names it.
build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: odigest})
bkey, _ := builds.BuilderBuildKey("oldie", ofp)
b.mu.Lock()
b.kv[builds.BuilderBuildsBucket+"/"+bkey] = build
b.mu.Unlock()
b.put(t, builds.BuilderBuildsBucket, bkey, build)
got := waitFor("oldie's done build", func(got []string) bool { return len(got) >= 3 })
if got[2] != oldieKey {
t.Errorf("the done build of oldie wrote %v, want %s", got[2:], oldieKey)
}
b.put(t, builds.BuilderBuildsBucket, bkey, build)
time.Sleep(100 * time.Millisecond)
if got := state.written(); len(got) != 3 {
t.Errorf("the same build landing again wrote %v", got[3:])
}
// A placed module whose newest build failed is published as a refusal at that build with the build's words (the
// provisioner holds it and does not wait); one whose build is queued or running stays unpublished.
for m, st := range map[string]builds.BuilderBuild{
"broken": {State: builds.BuildFailed, Error: `refused kind: package of language "go"`},
"waiting": {State: builds.BuildQueued},
} {
f := map[string]string{"broken": bfp, "waiting": wfp}[m]
raw, _ := builds.EncodeBuilderBuild(st)
k, _ := builds.BuilderBuildKey(m, f)
b.mu.Lock()
b.kv[builds.BuilderBuildsBucket+"/"+k] = raw
b.mu.Unlock()
b.put(t, builds.BuilderBuildsBucket, k, raw)
}
brokenKey, _ := engcomp.Key("n1", "broken", bfp)
got = waitFor("broken's failed build", func(got []string) bool { return len(got) >= 4 })
// The pass that wrote it says so ("1 key(s) written" was said once before, for oldie); then only brokenKey is new.
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
n := 0
b.mu.Lock()
for _, l := range b.said {
if strings.Contains(l, "this node's composition state: 1 key(s) written") {
n++
}
}
b.mu.Unlock()
if n >= 2 {
break
}
}
if got := state.written(); len(got) != 4 || got[3] != brokenKey {
t.Errorf("a failed build and a queued one wrote %v, want only %s", got[3:], brokenKey)
}
if comp, err := engcomp.Decode(state.kv[brokenKey].Value); err != nil || comp.Refused != `refused kind: package of language "go"` || len(comp.Resources) != 0 {
t.Errorf("the failed build's key: %+v, %v", comp, err)
}
// What is kept: the five keys of a module written last, and one a want names; the rest is deleted.
keep := New(b.env())
keep.pubKept = map[string]ComposedEntry{}
@@ -225,4 +230,47 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
if past := keep.pastKeeping(); len(past) != 0 {
t.Errorf("past keeping with a want naming the oldest: %v, want none", past)
}
brokenKey, _ := engcomp.Key("n1", "broken", bfp)
// Design 47 G A-293: reading the catalog half by fingerprint, a placed module's newest build that ended is the last trunk
// build.ended heard of it; a failed one is said refused with the event's words once the first read ended, an end naming a
// branch moves nothing, a module with no end (queued or running) stays unpublished, and a later done end, at any
// fingerprint, deletes the refused key.
byEnv := b.env()
byEnv.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
return nil, errors.New("nothing is asked")
})
byState := &fakeComposed{kv: map[string]ComposedEntry{}}
byEnv.ComposedState = byState
by := New(byEnv)
byAt := &published{fps: map[string]string{}, placed: []string{"broken", "waiting", "branched"}}
words := `refused kind: package of language "go"`
by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: bfp, Outcome: builds.BuildFailed, Error: words})
by.NoteBuildEnded(builds.BuildEnded{Module: "branched", Fingerprint: wfp, Outcome: builds.BuildFailed, Error: "no", Branch: "feature"})
for _, step := range []struct {
name string
do func()
holds map[string]string // by key, the refusal it says
}{
{"before the first read ended", func() {}, map[string]string{}},
{"once the first read ended", by.BuildEndsRead, map[string]string{brokenKey: words}},
{"a later done end", func() {
by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: wfp, Outcome: builds.BuildDone})
}, map[string]string{}},
} {
step.do()
if _, _, err := by.writeComposed(ctx, byAt); err != nil {
t.Fatalf("%s: %v", step.name, err)
}
byState.mu.Lock()
got := map[string]string{}
for k, v := range byState.kv {
comp, _ := engcomp.Decode(v.Value)
got[k] = comp.Refused
}
byState.mu.Unlock()
if !maps.Equal(got, step.holds) {
t.Errorf("%s: the state holds %v, want %v", step.name, got, step.holds)
}
}
}
internal/selfcompose/data_test.go
+1 -39
View File
@@ -5,8 +5,7 @@ import (
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"github.com/novox/mesh-host/internal/buckets"
engcomp "github.com/novox/mesh-host/internal/compositions"
"slices"
@@ -17,8 +16,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-host/internal/compose/frombus"
"github.com/novox/mesh-host/internal/agent"
)
// fakeData is the engine's state data as a map, counting the writes and deletes made.
@@ -231,39 +228,4 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) {
if wrote, _ := pc.WritePackages(context.Background()); wrote {
t.Error("the verb's write of a key holding what was composed wrote it again")
}
// The items are worked out from what the pass read: a done build is read over the bus once, not again for the data
// items after the pass (a module in a running build is read again, since it may land).
b := newFakeBus()
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte(lemurs))})
running, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"})
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
doneKey, _ := builds.BuilderBuildKey("web", fp)
runningKey, _ := builds.BuilderBuildKey("pg", fp)
b.kv[builds.BuilderBuildsBucket+"/"+doneKey], b.kv[builds.BuilderBuildsBucket+"/"+runningKey] = done, running
r := reader{c: New(b.env())}
for range 3 {
if got, err := r.Build(context.Background(), "web", fp); err != nil || got.State != builds.BuildDone {
t.Fatalf("the done build: %+v, %v", got, err)
}
if got, err := r.Build(context.Background(), "pg", fp); err != nil || got.State != "building" {
t.Fatalf("the running build: %+v, %v", got, err)
}
}
reads := map[string]int{}
for _, read := range b.reads {
reads[read[1]]++
}
if reads[doneKey] != 1 || reads[runningKey] != 3 {
t.Errorf("the builds were read %v times over the bus; want the done one once, the running one each time", reads)
}
// A done key the builder claims again (a gone step, issue 1937) and lands done again is read again, not served kept.
again, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte("{}"))})
b.mu.Lock()
b.kv[builds.BuilderBuildsBucket+"/"+doneKey] = again
b.mu.Unlock()
r.c.takeBuildDone(agent.Update{Key: doneKey, Value: running})
if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != meshcatalogclient.ManifestDigest([]byte("{}")) {
t.Errorf("a done build written again is served as %+v (%v), not read again", got, err)
}
}
internal/selfcompose/entries.go
+653
View File
@@ -0,0 +1,653 @@
package selfcompose
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"maps"
"reflect"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-host/internal/agent"
)
// **The catalog half, read by fingerprint** (design 47 G §421, §683, §698; the lead's answers A-47 and A-50). The node-engine
// follows no branch: for each build its wants and the runnings it heard name, it holds the catalog graph's entry of that
// fingerprint, asked of the seat mesh-catalog's verb `entry` once (meshcatalogclient.Entries) and kept in its composition
// state `composed-<node>` beside the compositions (`entry.<fingerprint>`, the entry as `entry` answers it), with each want it
// composes from (`want.<module>`), written whole before the composition that uses them and read at its start before it asks
// anything. A module's manifest is the entry's, and each artifact's reference is made from the digest its want's parts name
// (referenceOf), so it reads neither the indexer's states nor the builder's.
// The keys of the kept entries and wants in the composition state.
const (
keptEntryPrefix = "entry."
keptWantPrefix = "want."
// keptWithin bounds one write of the composition state from an answer of the catalog half.
keptWithin = 15 * time.Second
)
// keptWant is a want as the composition state keeps it (§683): the want beside when this node first kept it.
type keptWant struct {
V int `json:"v"`
Since time.Time `json:"since"`
Want meshwants.WantsByWantNodeModuleValue `json:"want"`
}
// keptEntry is an entry held from the composition state or an answer, with when it was kept.
type keptEntry struct {
entry meshcatalog.CatalogEntry
since time.Time
}
// EntryRow is one entry the node-engine keeps, as the seat's verb first-views lists it (§698).
type EntryRow struct {
Fingerprint string `json:"fingerprint"`
State string `json:"state"`
Since time.Time `json:"since"`
}
// WantRow is one want the node-engine keeps, as the seat's verb first-views lists it (§698).
type WantRow struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
KeptSince time.Time `json:"kept-since"`
}
// newEntries makes the composer's cache of entries over env's call of `entry`.
func (c *Composer) newEntries() {
if c.env.Entry == nil {
return
}
c.entries = meshcatalogclient.NewEntries(c.env.Entry)
c.entries.Now = c.now
c.entries.Changed = c.entryChanged
c.keptEnt = map[string]keptEntry{}
c.keptWants = map[string]bool{}
c.keptWantVal = map[string]keptWant{}
}
// entryKey is an entry's key in the composition state, wantKey a want's.
//
// The fingerprint is keyed by its digits, as the compositions' keys are (engcomp.Key): a key holds no colon.
func entryKey(fp string) string {
return keptEntryPrefix + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
}
func wantStateKey(module string) string { return keptWantPrefix + module }
// loadKept reads the entries and wants the composition state keeps, once at the start, before anything is asked of the catalog
// half (§683). A want read stands only until the live read of the wants answers (endKeptWants). A state that cannot be read
// is said, and what it would have kept is asked of `entry` and read from the wants as they come.
func (c *Composer) loadKept(ctx context.Context) {
if c.env.ComposedState == nil {
return
}
kept, err := c.env.ComposedState.Entries(ctx)
if err != nil {
c.sayOnceEach("the entries and wants this node kept cannot be read, so each entry its wants name is asked of the catalog half: " + err.Error())
return
}
now := c.now()
c.mu.Lock()
defer c.mu.Unlock()
for key, e := range kept {
switch {
case strings.HasPrefix(key, keptEntryPrefix):
var a meshcatalog.EntryAnswer
if json.Unmarshal(e.Value, &a) != nil || a.Entry.Module == "" {
c.env.Say("the kept entry " + key + " cannot be read, and is asked of the catalog half again")
continue
}
c.keptEnt[meshcatalogclient.SourcePrefix2+strings.TrimPrefix(key, keptEntryPrefix)] = keptEntry{entry: a.Entry, since: now}
case strings.HasPrefix(key, keptWantPrefix):
var w keptWant
module := strings.TrimPrefix(key, keptWantPrefix)
if json.Unmarshal(e.Value, &w) != nil || w.Want.Fp == "" {
c.env.Say("the kept want " + key + " cannot be read, so it stands until the wants are read")
continue
}
if _, live := c.wants[module]; !live {
c.wants[module] = w.Want
c.keptWants[module] = true
}
c.keptWantVal[module] = w
}
}
}
// endKeptWants ends the standing of the kept wants once the live read of the wants answered: from then the wants the seat
// says are the ones the node composes from, never a want older than its stream's (§683). A kept want the read did not
// announce is gone.
func (c *Composer) endKeptWants() {
c.mu.Lock()
defer c.mu.Unlock()
for module := range c.keptWants {
delete(c.wants, module)
delete(c.keptWants, module)
}
}
// entryChanged is told of each entry answered or found missing: an answer is kept in the composition state before the pass it
// asks (so the composition that uses it finds it kept), and a pass is asked.
func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) {
if row.State == meshcatalogclient.EntryAnswered && row.Entry != nil {
c.mu.Lock()
c.keptEnt[row.Fingerprint] = keptEntry{entry: *row.Entry, since: row.Since}
c.mu.Unlock()
c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry})
}
c.tellNotFound(row.Fingerprint, row.State == meshcatalogclient.EntryNotFound)
c.askRehand()
}
// tellNotFound tells the finding entry-not-found once when fp's entry is not found, and once when it no longer is (design 47 G §614, A-117).
func (c *Composer) tellNotFound(fp string, notFound bool) {
c.mu.Lock()
was := c.notFound[fp]
if was == notFound {
c.mu.Unlock()
return
}
if c.notFound == nil {
c.notFound = map[string]bool{}
}
if notFound {
c.notFound[fp] = true
} else {
delete(c.notFound, fp)
}
c.mu.Unlock()
if c.env.EntryNotFound != nil {
c.env.EntryNotFound(fp, notFound)
}
}
// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by
// the next pass.
func (c *Composer) keep(key string, v any) {
if c.env.ComposedState == nil {
return
}
raw, err := json.Marshal(v)
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), keptWithin)
defer cancel()
c.keptWrite.Lock()
defer c.keptWrite.Unlock()
if _, err := c.env.ComposedState.Put(ctx, key, raw); err != nil {
c.sayOnceEach("the node's composition state cannot keep " + key + ": " + err.Error())
}
}
// forget deletes one key from the composition state.
func (c *Composer) forget(key string) {
if c.env.ComposedState == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), keptWithin)
defer cancel()
c.keptWrite.Lock()
defer c.keptWrite.Unlock()
if err := c.env.ComposedState.Delete(ctx, key); err != nil {
c.sayOnceEach("the node's composition state cannot drop " + key + ": " + err.Error())
}
}
// entryOf is the entry of fp as the composer holds it: kept, or asked of the catalog half once (Entries.Need).
func (c *Composer) entryOf(fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) {
c.mu.Lock()
k, ok := c.keptEnt[fp]
c.mu.Unlock()
if ok {
return k.entry, meshcatalogclient.EntryAnswered
}
return c.entries.Need(context.Background(), fp)
}
// wantedBuilds is each fingerprint a want of this node or a running heard names, of the new form, with whether a want names it.
func (c *Composer) wantedBuilds() map[string]bool {
out := map[string]bool{}
c.mu.Lock()
defer c.mu.Unlock()
for module, w := range c.wants {
if module != agent.NodeModule && strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) {
out[w.Fp] = true
}
}
for _, mods := range c.heard {
for _, run := range mods {
if !run.stopped && strings.HasPrefix(run.fingerprint, meshcatalogclient.SourcePrefix2) {
if _, named := out[run.fingerprint]; !named {
out[run.fingerprint] = false
}
}
}
}
return out
}
// syncEntries asks the entry of each build a want or a running names, drops the one nothing names any more, keeps each want it
// composes from, and lays the picture frombus reads (c.modules, c.manifests) from the entries it holds. It runs at the start of
// each pass.
func (c *Composer) syncEntries(ctx context.Context) {
need := c.wantedBuilds()
for fp := range need {
c.entryOf(fp)
}
for _, row := range c.entries.Rows() {
c.mu.Lock()
added := c.addedFps[row.Fingerprint]
c.mu.Unlock()
if _, named := need[row.Fingerprint]; !named && !added {
c.entries.Release(row.Fingerprint)
c.tellNotFound(row.Fingerprint, false)
}
}
c.mu.Lock()
var drop []string
for fp := range c.keptEnt {
if _, named := need[fp]; !named && !c.addedFps[fp] {
delete(c.keptEnt, fp)
drop = append(drop, entryKey(fp))
}
}
type pending struct {
module string
kw keptWant
}
var put []pending
var gone []string
now := c.now()
for module, w := range c.wants {
if module == agent.NodeModule || c.keptWants[module] {
continue
}
had, ok := c.keptWantVal[module]
if ok && reflect.DeepEqual(had.Want, w) {
continue
}
since := now
if ok && had.Want.Fp == w.Fp {
since = had.Since
}
kw := keptWant{V: 1, Since: since, Want: w}
c.keptWantVal[module] = kw
put = append(put, pending{module, kw})
}
for module := range c.keptWantVal {
if _, held := c.wants[module]; !held {
delete(c.keptWantVal, module)
gone = append(gone, wantStateKey(module))
}
}
c.mu.Unlock()
// Written whole before the composition that uses them (§683).
for _, p := range put {
c.keep(wantStateKey(p.module), p.kw)
}
for _, key := range append(drop, gone...) {
c.forget(key)
}
c.layModules()
}
// layModules lays c.modules and c.manifests from the entries held: what each node was heard to run first, then each module this
// node's want names, at the want's fingerprint (a want wins).
func (c *Composer) layModules() {
c.mu.Lock()
defer c.mu.Unlock()
modules := map[string]meshcatalog.ModulesByModuleValue{}
lay := func(module, fp string) {
k, ok := c.keptEnt[fp]
if !ok || k.entry.Module == "" {
return
}
e := k.entry
digest, raw, err := entryManifest(e)
if err != nil {
return
}
m := meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: e.Repository, Commit: e.Commit, Path: e.Path,
Needs: e.Needs, Packages: e.Packages, Verdict: e.Verdict}
if e.Inputs != nil {
m.Problems, m.Warnings, m.Unknown = e.Inputs.Problems, e.Inputs.Warnings, e.Inputs.Unknown
}
modules[module] = m
c.manifests[digest] = raw
}
nodes := make([]string, 0, len(c.heard))
for node := range c.heard {
nodes = append(nodes, node)
}
sort.Strings(nodes)
for _, node := range nodes {
for module, run := range c.heard[node] {
if !run.stopped && run.fingerprint != "" {
lay(module, run.fingerprint)
}
}
}
for module, w := range c.wants {
if module != agent.NodeModule && w.Fp != "" {
lay(module, w.Fp)
}
}
// A module an assignment places whose entry is not held (not wanted here, not heard running, or its entry asked and not
// answered) stands in the catalogue as its name alone, so the placement still finds it: a module held for its entry offers
// nothing to the rest until its entry is held (§421).
for _, p := range c.assignments {
for _, pm := range p.Modules {
if _, held := modules[pm.Module]; held || pm.Module == "" {
continue
}
raw, _ := json.Marshal(map[string]any{"module": pm.Module, "version": "0"})
sum := sha256.Sum256(raw)
digest := "sha256:" + hex.EncodeToString(sum[:])
// At the fingerprint its want names, so it is no build behind the catalog's and its hold (entryHolds) is what says it.
modules[pm.Module] = meshcatalog.ModulesByModuleValue{Manifest: digest, Fp2: c.wants[pm.Module].Fp}
c.manifests[digest] = raw
}
}
c.modules = modules
}
// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set
// none, the digest of the manifest's bytes.
func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) {
if raw, err = json.Marshal(e.Manifest); err != nil {
return "", nil, err
}
digest = e.ManifestDigest
if digest == "" {
sum := sha256.Sum256(raw)
digest = "sha256:" + hex.EncodeToString(sum[:])
}
return digest, raw, nil
}
// entryHolds is, of each module this node's want names, why its entry is not held: asked and not answered, or answered not
// found. It is the phase held with a detail opening entry-asked: or entry-missing: and naming the fingerprint (§421, §614).
func (c *Composer) entryHolds() map[string]string {
c.mu.Lock()
wants := map[string]meshwants.WantsByWantNodeModuleValue{}
for module, w := range c.wants {
if module != agent.NodeModule {
wants[module] = w
}
}
c.mu.Unlock()
out := map[string]string{}
for module, w := range wants {
if !strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) {
continue
}
switch e, state := c.entryOf(w.Fp); state {
case meshcatalogclient.EntryAsked:
out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s",
agent.EntryAsked, w.Fp, module)
case meshcatalogclient.EntryNotFound:
out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s",
agent.EntryMissing, w.Fp, module)
default:
if missing := partsMissing(e, w); missing != "" {
out[module] = fmt.Sprintf("the want of %s names no digest for %s, so the node composes nothing new of it", module, missing)
}
}
}
return out
}
// partsMissing names the artifacts of e whose digest w's parts do not give; "" when each has one.
func partsMissing(e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) string {
var out []string
for _, a := range e.Artifacts {
if w.Parts[a.Ref] == "" {
out = append(out, a.Ref)
}
}
sort.Strings(out)
return strings.Join(out, ", ")
}
// referenceOf is an artifact's reference made from the digest its want's part names, in the form the artifact store is
// addressed by (the lead's answer A-50): artifact-store://<module>/<artifact>/blobs/<digest>.
func referenceOf(module, artifact, digest string) string {
return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
}
// buildOf is the build a want names, made from the entry the composer holds and the parts of the want (the builder's done key is
// not read, §421, §628): done, with each artifact of the entry at the digest its part names and the reference made from it. Nil
// where the entry is not held, the want names another fingerprint, or a part is missing. The shadow's wants for the modules a
// candidate adds carry the parts the candidate gives (A-316).
func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild {
c.mu.Lock()
w, wanted := c.wants[module]
k, ok := c.keptEnt[fp]
c.mu.Unlock()
if !wanted || w.Fp != fp || !ok || partsMissing(k.entry, w) != "" {
return nil
}
digest, raw, err := entryManifest(k.entry)
if err != nil {
return nil
}
c.mu.Lock()
c.manifests[digest] = raw
c.mu.Unlock()
b := builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}
for _, a := range k.entry.Artifacts {
_, name, _ := strings.Cut(a.Ref, ":")
digest := w.Parts[a.Ref]
b.Artifacts = append(b.Artifacts, builds.BuiltArtifact{Name: name, Kind: a.Kind, Digest: digest,
Reference: referenceOf(module, name, digest)})
}
return &b
}
// FirstViews is the node-engine's first views as the seat's verb first-views lists them (§698): the views it follows (none: it
// follows no branch), the entries it keeps and the wants it composes from.
func (c *Composer) FirstViews() (views []map[string]any, entries []EntryRow, wants []WantRow) {
views, entries, wants = []map[string]any{}, []EntryRow{}, []WantRow{}
seen := map[string]bool{}
for _, r := range c.entries.Rows() {
seen[r.Fingerprint] = true
entries = append(entries, EntryRow{Fingerprint: r.Fingerprint, State: string(r.State), Since: r.Since})
}
c.mu.Lock()
for fp, k := range c.keptEnt {
if !seen[fp] {
entries = append(entries, EntryRow{Fingerprint: fp, State: string(meshcatalogclient.EntryAnswered), Since: k.since})
}
}
for module, w := range c.wants {
if module == agent.NodeModule || w.Fp == "" {
continue
}
wants = append(wants, WantRow{Module: module, Fingerprint: w.Fp, KeptSince: c.keptWantVal[module].Since})
}
c.mu.Unlock()
sort.Slice(entries, func(i, j int) bool { return entries[i].Fingerprint < entries[j].Fingerprint })
sort.Slice(wants, func(i, j int) bool { return wants[i].Module < wants[j].Module })
return
}
// EntryAgain asks fp's entry of the catalog half again now (the seat's verb entry-again, §698): one the composer keeps, whatever
// its state. The entry kept of it is dropped, so a pass holds the module until the answer.
func (c *Composer) EntryAgain(ctx context.Context, fp string) error {
if c.AlsoEntryAgain != nil {
c.AlsoEntryAgain(ctx, fp)
}
c.mu.Lock()
_, kept := c.keptEnt[fp]
delete(c.keptEnt, fp)
c.mu.Unlock()
held := kept
for _, r := range c.entries.Rows() {
held = held || r.Fingerprint == fp
}
if !held {
return fmt.Errorf("no entry of %s is kept", fp)
}
if !kept {
return c.entries.Again(ctx, fp)
}
c.entries.Need(ctx, fp) // makes the row and asks once
c.askRehand()
return nil
}
// EntryCall is env.Entry over the engine's link session: the seat mesh-catalog's verb `entry`, read in the engine's own name
// (the manifest's invokes names it). A refusal in the verb's own words that it keeps no entry of the build is the holder's
// answer that it holds none, and is told so; any other failure is a call that did not reach the holder, and is told so too.
func EntryCall(ask func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error)) meshcatalogclient.Call {
return func(key string, body any) (json.RawMessage, error) {
seatName, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
raw, err := ask(ctx, seatName, verb, body)
if err != nil && strings.Contains(err.Error(), noEntryWords) {
out, _ := json.Marshal(map[string]string{"error": err.Error()})
return out, nil
}
return raw, err
}
}
// noEntryWords are the words the seat's holder refuses an unknown fingerprint in.
const noEntryWords = "no entry of fingerprint"
// FormChanges are the kept data whose form differs between the manifests of the build a module runs and the one a want moves it
// to, each read from the entry the composer holds of its fingerprint (design 47 G §296, §297). An entry not held is an error: the
// node agent then says it is not known and the move goes on.
func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error) {
read := func(fp string) (catalogue.Manifest, error) {
e, state := c.entryOf(fp)
if state != meshcatalogclient.EntryAnswered {
return catalogue.Manifest{}, fmt.Errorf("the entry of %s is %s", fp, state)
}
_, raw, err := entryManifest(e)
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.Decode(raw)
}
running, err := read(runningFp)
if err != nil {
return nil, fmt.Errorf("%s's running manifest: %w", module, err)
}
applied, err := read(appliedFp)
if err != nil {
return nil, fmt.Errorf("%s's wanted manifest: %w", module, err)
}
return catalogue.FormChanges(running, applied), nil
}
// syncShadowEntries asks the entry of each build a want of this node names, drops the one nothing names, lays c.modules and
// c.manifests from the entries held, and says whether an entry asked is not yet answered: the shadow's picture of the catalog,
// where it once watched the indexer's modules (design 47 G §421, A-308).
func (c *Composer) syncShadowEntries(ctx context.Context) (pending bool) {
c.syncEntries(ctx)
c.mu.Lock()
defer c.mu.Unlock()
for module, w := range c.wants {
if module == agent.NodeModule || !strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) {
continue
}
if _, held := c.keptEnt[w.Fp]; !held {
for _, r := range c.entries.Rows() {
if r.Fingerprint == w.Fp && r.State == meshcatalogclient.EntryAsked {
return true
}
}
}
}
return false
}
// moduleVerb is the seat mesh-catalog's verb `module`: one module's trunk fingerprint and manifest by name.
const moduleVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbModule
// addedModule is the trunk's build of a module an assignment candidate adds to this node when no want here names it yet: the
// fingerprint the seat's verb `module` answers, then its entry by the verb `entry` as for a wanted module (design 47 G §421, A-312).
// A call, not a watch: nothing of the catalog is kept. It answers the entry when held; otherwise why not: a module with no
// fingerprint says the verb's reason, an entry asked says so (its answer wakes the shadow, which composes again).
func (c *Composer) addedModule(ctx context.Context, module string) (fp string, entry meshcatalog.CatalogEntry, held, pending bool, why string) {
if c.env.Entry == nil {
return "", entry, false, false, "this node reads no module of the catalog half"
}
raw, err := c.env.Entry(moduleVerb, meshcatalog.ModuleInput{Module: module})
if err != nil {
return "", entry, false, false, fmt.Sprintf("the catalog half's answer for %s: %v", module, err)
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return "", entry, false, false, fmt.Sprintf("the catalog half holds no module %s: %s", module, said)
}
var a meshcatalog.ModuleAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return "", entry, false, false, fmt.Sprintf("the catalog half's answer for %s cannot be read: %v", module, err)
}
if a.Module.Fp == "" {
return "", entry, false, false, fmt.Sprintf("the catalog half gives %s no fingerprint: %s", module, a.Module.None)
}
fp = a.Module.Fp
c.mu.Lock()
if c.addedFps == nil {
c.addedFps = map[string]bool{}
}
c.addedFps[fp] = true
c.mu.Unlock()
e, state := c.entryOf(fp)
switch state {
case meshcatalogclient.EntryAnswered:
return fp, e, true, false, ""
case meshcatalogclient.EntryNotFound:
return fp, e, false, false, fmt.Sprintf("the catalog half holds no entry of %s's build %s", module, fp)
}
return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp)
}
// askBuildsEntries asks the entry of each build in fps, the builds the node runs or its wants name, that the composer does not hold,
// and says the first not yet answered: the shadow composes a candidate from entries alone (§421, §628), and an answer wakes it. The
// fingerprints are kept beside the candidates' added ones while a proposal stands (addedFps).
func (c *Composer) askBuildsEntries(fps map[string]string) (pending string) {
for _, module := range slices.Sorted(maps.Keys(fps)) {
fp := fps[module]
if !strings.HasPrefix(fp, meshcatalogclient.SourcePrefix2) {
continue
}
c.mu.Lock()
if c.addedFps == nil {
c.addedFps = map[string]bool{}
}
c.addedFps[fp] = true
c.mu.Unlock()
e, state := c.entryOf(fp)
switch {
case state == meshcatalogclient.EntryAnswered:
c.mu.Lock()
if _, kept := c.keptEnt[fp]; !kept && e.Module != "" {
c.keptEnt[fp] = keptEntry{entry: e, since: c.now()}
}
c.mu.Unlock()
case state == meshcatalogclient.EntryAsked && pending == "":
pending = fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp)
}
}
return pending
}
internal/selfcompose/entries_test.go
+261
View File
@@ -0,0 +1,261 @@
package selfcompose
import (
"context"
"encoding/json"
"errors"
"fmt"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/buckets"
"github.com/novox/mesh-host/internal/buckets/bucketstest"
"github.com/novox/mesh-host/internal/compositions"
)
// Design 47 G §421, §683, §698 (the lead's answers A-47, A-50): the node-engine reads the entry its want names by the
// fingerprint, through its own calls of the seat mesh-catalog's verb entry, and reads neither the indexer's states nor the
// builder's; a want whose entry the catalog half holds none of is held, its have saying entry-missing and naming the
// fingerprint; the entries and wants it composes from are kept in its composition state, written before the composition that
// uses them and read at its start before anything is asked; a want removed takes its kept want and entry away.
func composeFromEntries(t *testing.T) {
b := newFakeBus()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
ghost := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
var manifest map[string]any
if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil {
t.Fatal(err)
}
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c",
ManifestDigest: meshcatalogclient.ManifestDigest([]byte(lemurs))}
var mu sync.Mutex
var asked []string
ask := func(_ context.Context, seat, verb string, args any) (json.RawMessage, error) {
in := args.(meshcatalog.EntryInput)
mu.Lock()
asked = append(asked, in.Fingerprint)
mu.Unlock()
if seat != "mesh-catalog" || verb != "entry" {
t.Errorf("asked %s.%s", seat, verb)
}
if in.Fingerprint != fp {
return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept")
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
}
state := &fakeComposed{kv: map[string]ComposedEntry{}}
var held map[string]string
env := b.env()
env.Entry, env.ComposedState = EntryCall(ask), state
refused := make(chan struct{}, 1) // fed at each refusal the composer says, so a wait follows what it says, not a pause
env.Refused = func(_ string, h map[string]string) {
mu.Lock()
held = h
mu.Unlock()
select {
case refused <- struct{}{}:
default:
}
}
told := map[string]bool{}
env.EntryNotFound = func(fp string, notFound bool) { mu.Lock(); told[fp] = notFound; mu.Unlock() }
c := New(env)
out, err := c.Watch(ctx, "n1.>")
if err != nil {
t.Fatal(err)
}
place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"},
Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "ghost"}}})
have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true})
b.putAssigned(t, place)
b.put(t, buckets.Have, "n1.node", have)
b.put(t, buckets.Wants, "want.n1.node", nodeWant)
for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: fp, Self: true}, "ghost": {Fp: ghost, Self: true}} {
raw, _ := bucketstest.Want(w)
b.put(t, buckets.Wants, "want.n1."+module, raw)
}
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} {
b.send(bucket, agent.Update{Caught: true})
}
lemursKey, _ := compositions.Key("n1", "lemurs", fp)
handed := map[string]bool{}
for !handed[lemursKey] {
handed[next(t, out).Key] = true
}
// Neither the indexer's states nor the builder's are watched or read.
b.mu.Lock()
for _, r := range b.reads {
if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || strings.HasSuffix(r[0], "_builds") {
t.Errorf("the node-engine read %s of %s", r[1], r[0])
}
}
b.mu.Unlock()
// A want whose entry the half holds none of is held, saying entry-missing and the fingerprint; the rest is composed.
// The ghost's entry is asked first and answered not found after: wait until the refusal says entry-missing.
var why string
for deadline := time.After(10 * time.Second); !strings.HasPrefix(why, agent.EntryMissing); {
mu.Lock()
why = held["ghost"]
mu.Unlock()
if strings.HasPrefix(why, agent.EntryMissing) {
break
}
select {
case <-refused:
case <-deadline:
t.Fatalf("the ghost's reason is still %q after 10 s, want %s", why, agent.EntryMissing)
}
}
if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" {
t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost)
}
// And the finding entry-not-found is told for the fingerprint the half holds none of, and for no other (design 47 G §614, A-117).
mu.Lock()
nf, nfOther := told[ghost], told[fp]
mu.Unlock()
if !nf || nfOther {
t.Errorf("entry-not-found told %v for the ghost and %v for the entry held", nf, nfOther)
}
// An entry asked again by hand is asked again in the shadow's cache too (lead answer A-319): the shadow holds the ghost as not
// found, the half then holds it, and the node's own composer asked again takes it in on the shadow as well.
var late atomic.Bool
shadowEnv := b.env()
shadowEnv.Entry = EntryCall(func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error) {
if in := args.(meshcatalog.EntryInput); in.Fingerprint == ghost && late.Load() {
return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "ghost", Fp: ghost}})
}
return ask(ctx, seat, verb, args)
})
shadow := NewShadow(shadowEnv, nil)
shadow.c.entries.Need(ctx, ghost)
for deadline := time.Now().Add(10 * time.Second); shadow.c.entries.Rows() == nil || shadow.c.entries.Rows()[0].State != meshcatalogclient.EntryNotFound; {
if time.Now().After(deadline) {
t.Fatal("the shadow did not hold the ghost as not found")
}
runtime.Gosched()
}
late.Store(true)
c.AlsoEntryAgain = shadow.EntryAgain
if err := c.EntryAgain(ctx, ghost); err != nil {
t.Fatal(err)
}
if _, state := shadow.c.entries.Need(ctx, ghost); state != meshcatalogclient.EntryAnswered {
t.Errorf("the shadow holds the ghost's entry as %s after entry-again, want answered", state)
}
// Each entry and want it composes from is kept in the composition state before the composition that uses them.
state.mu.Lock()
_, hasEntry := state.kv[entryKey(fp)]
_, hasWant := state.kv["want.lemurs"]
state.mu.Unlock()
if !hasEntry || !hasWant {
t.Errorf("the composition state holds entry %v and want %v of lemurs: %v", hasEntry, hasWant, state.written())
}
// The seat's first-views lists them.
_, entries, wants := c.FirstViews()
if len(entries) < 2 || len(wants) != 2 {
t.Errorf("first-views lists %v and %v", entries, wants)
}
// A restart reads what is kept before it asks anything: lemurs's entry is not asked again.
mu.Lock()
asked = nil
mu.Unlock()
env2 := b.env()
env2.Entry, env2.ComposedState = EntryCall(ask), state
// The node's own record, stated by the node-engine of itself in memory (A-120).
env2.NodeRecord = func() (nodeengine.NodeRecord, bool) {
return nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}, true
}
c2 := New(env2)
out2, err := c2.Watch(ctx, "n1.>")
if err != nil {
t.Fatal(err)
}
// And composes the module it kept a want and an entry of before any input is read, nothing of the others (design 47 G §683, A-117).
if u := next(t, out2); u.Key != lemursKey || u.Deleted {
t.Errorf("the early pass handed %q, want the composition %s of lemurs", u.Key, lemursKey)
}
c2.mu.Lock()
kept, keptWant := c2.keptEnt[fp], c2.wants["lemurs"].Fp
c2.mu.Unlock()
if kept.entry.Module != "lemurs" || keptWant != fp {
t.Errorf("a restart read entry %q and want %q from the state", kept.entry.Module, keptWant)
}
// A want removed takes its kept want and entry away.
raw, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Removed: true})
b.put(t, buckets.Wants, "want.n1.lemurs", raw)
for range 20 {
state.mu.Lock()
_, hasWant = state.kv["want.lemurs"]
_, hasEntry = state.kv[entryKey(fp)]
state.mu.Unlock()
if !hasWant && !hasEntry {
break
}
next(t, out)
}
if hasWant || hasEntry {
t.Errorf("a removed want left its kept want %v and entry %v", hasWant, hasEntry)
}
}
// catalogHalf is the catalog half's entries a test composes from, by fingerprint, answered through the seat mesh-catalog's verbs
// entry and module (design 47 G §421, A-312): the node-engine reads no indexer.
type catalogHalf map[string]meshcatalog.CatalogEntry
// add holds the entry of module at fp, made of the manifest body.
func (h catalogHalf) add(t *testing.T, module, fp, body string) {
t.Helper()
var m map[string]any
if err := json.Unmarshal([]byte(body), &m); err != nil {
t.Fatal(err)
}
h[fp] = meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, Repository: "novox/" + module, Commit: "c", ManifestDigest: meshcatalogclient.ManifestDigest([]byte(body))}
}
func (h catalogHalf) call() meshcatalogclient.Call {
return EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) {
switch in := args.(type) {
case meshcatalog.ModuleInput:
for _, e := range h {
if e.Module == in.Module {
return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: e.Module, Repository: e.Repository, Commit: e.Commit, Fp: e.Fp}})
}
}
return nil, fmt.Errorf("the catalog holds no module %s", in.Module)
case meshcatalog.EntryInput:
if e, ok := h[in.Fingerprint]; ok {
return json.Marshal(meshcatalog.EntryAnswer{Entry: e})
}
return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept")
}
return nil, fmt.Errorf("asked %s of %T", verb, args)
})
}
// holdEntry makes the composer hold the entry of module at fp, as its call of the verb entry would have answered it (§421), for the
// tests that set the rest of its picture directly.
func (c *Composer) holdEntry(t *testing.T, module, fp, body string) {
t.Helper()
half := catalogHalf{}
half.add(t, module, fp, body)
c.mu.Lock()
defer c.mu.Unlock()
if c.keptEnt == nil {
c.keptEnt = map[string]keptEntry{}
}
c.keptEnt[fp] = keptEntry{entry: half[fp]}
}
internal/selfcompose/kept.go
+31
View File
@@ -0,0 +1,31 @@
package selfcompose
import (
"context"
"strings"
)
// KeptBuildOf is the node that keeps the build a module is wanted at, and its fingerprint, for the artifact store's
// holder (design 47 G section 271, 526, A-50): the want's `kept-on` and nothing else names it, the builder's builds being
// no read of the node-engine's (section 421). kept is false when the module has no want, its want names no keeping node,
// or its `parts` do not name the digest ("sha256:" and 64 hex digits, or bare): the artifact is then fetched from the store.
func (c *Composer) KeptBuildOf(_ context.Context, module, digest string) (node, fp string, kept bool, err error) {
c.mu.Lock()
w, ok := c.wants[module]
c.mu.Unlock()
if !ok || w.Fp == "" || w.KeptOn == "" {
return "", "", false, nil
}
if len(w.Parts) > 0 {
named := false
for _, d := range w.Parts {
if strings.TrimPrefix(d, "sha256:") == strings.TrimPrefix(digest, "sha256:") {
named = true
}
}
if !named {
return "", "", false, nil
}
}
return w.KeptOn, w.Fp, true, nil
}

Some files were not shown because too many files have changed in this diff Show More