Compare commits
43
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32c5df952c | ||
|
|
4363c371c5 | ||
|
|
1b01d065ac | ||
|
|
d3ee49d6f3 | ||
|
|
ded92a8137 | ||
|
|
20614822df | ||
|
|
9740238ec9 | ||
|
|
a2313057c0 | ||
|
|
2b14f166de | ||
|
|
d8207358b0 | ||
|
|
c6ae0fc405 | ||
|
|
0128b962ab | ||
|
|
271de0108d | ||
|
|
c93feeb30f | ||
|
|
7f33c8b499 | ||
|
|
18c1735f7c | ||
|
|
2a71a8ff76 | ||
|
|
e73aae3686 | ||
|
|
3eef84d979 | ||
|
|
60358d01e9 | ||
|
|
05418b4fe5 | ||
|
|
2779f5312b | ||
|
|
e2380404a1 | ||
|
|
998d8f7370 | ||
|
|
5dc3a8e17f | ||
|
|
768f637bd0 | ||
|
|
ee0f46b78f | ||
|
|
84a14b08d5 | ||
|
|
fabfdf04f7 | ||
|
|
966093d8d0 | ||
|
|
4d47ff5ce7 | ||
|
|
8e32583089 | ||
|
|
6019c6d514 | ||
|
|
054754482c | ||
|
|
f7129fdb41 | ||
|
|
f7abd05981 | ||
|
|
514cf23aef | ||
|
|
95b509e0d2 | ||
|
|
83e135f101 | ||
|
|
30b86ea602 | ||
|
|
32a99144f6 | ||
|
|
3544d644ba | ||
|
|
a0ce37fb2c |
@@ -4,3 +4,5 @@
|
||||
# The placeholder `make bootstrap` moves aside while a saved image is embedded. Ignored so an
|
||||
# interrupted release build cannot commit a twenty-megabyte tar by accident.
|
||||
/internal/image/builder.tar.placeholder
|
||||
/cmd/mesh-host/mesh-host
|
||||
/cmd/mesh-bootstrap/mesh-bootstrap
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshvault"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
@@ -108,13 +109,13 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
answer := signedBy(vaultCA, vault)
|
||||
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
|
||||
asked = append(asked, verb, args)
|
||||
return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||
return json.Marshal(meshvault.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||
}
|
||||
written := 0
|
||||
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb {
|
||||
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != meshvault.VerbSign {
|
||||
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
|
||||
}
|
||||
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
|
||||
|
||||
@@ -6,10 +6,13 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
@@ -85,4 +88,62 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
||||
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
|
||||
// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
|
||||
told := &settledRecorder{}
|
||||
events := make(chan []byte, 4)
|
||||
for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
|
||||
events <- []byte(body)
|
||||
}
|
||||
linked, end := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for len(told.heard()) < 2 && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
end()
|
||||
<-done
|
||||
if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
|
||||
t.Errorf("the agent was told %q settled, want distribution,gitea", got)
|
||||
}
|
||||
// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
|
||||
// one (ENG-8, ENG-10).
|
||||
d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
|
||||
`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pull := func(image string) error {
|
||||
return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
|
||||
}
|
||||
if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
|
||||
t.Errorf("a store's image is said as %+v", got)
|
||||
}
|
||||
if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
|
||||
t.Errorf("a public image is said as %+v", got)
|
||||
}
|
||||
if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
|
||||
t.Errorf("a failure that is no pull is said as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// settledRecorder is the agent a hearing tells.
|
||||
type settledRecorder struct {
|
||||
mu sync.Mutex
|
||||
list []string
|
||||
}
|
||||
|
||||
func (r *settledRecorder) Settled(holder string) {
|
||||
r.mu.Lock()
|
||||
r.list = append(r.list, holder)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *settledRecorder) heard() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.list...)
|
||||
}
|
||||
|
||||
+14
-14
@@ -13,10 +13,10 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/accounts"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
@@ -255,9 +255,9 @@ func engineActs(opts options, node string, queue *link.Session, search *accounts
|
||||
})
|
||||
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
|
||||
},
|
||||
Say: queue.SayEngine}
|
||||
asker := asks.EngineAsker(node)
|
||||
h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||
ManifestReads: queue.StatedReads, Say: queue.SayEngine}
|
||||
asker := occ.EngineAsker(node)
|
||||
h.Asker = &occ.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
|
||||
return h
|
||||
}
|
||||
@@ -364,26 +364,26 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
|
||||
|
||||
// composeCandidateFrom answers the verb compose-candidate (novox/hq ADR 0593 §7): the check's candidate composed on this
|
||||
// node in shadow by the run's shadow, applying and writing nothing, within the shadow's own bound (shorter than
|
||||
// delivery.ComposeAnswerBound, selfcompose candidateWithin). The answer is
|
||||
// the seat's shape (delivery.ComposeCandidateAnswer), checked as it is encoded; a node whose shadow does not run yet
|
||||
// nec.AnswerBound, selfcompose candidateWithin). The answer is
|
||||
// the seat's shape (nodeengine.ComposeCandidateAnswer), checked as it is encoded; a node whose shadow does not run yet
|
||||
// answers unready, which the checker takes as no answer.
|
||||
func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow]) func(candidate string, input bool) (any, error) {
|
||||
return func(candidate string, input bool) (any, error) {
|
||||
ask := delivery.ComposeCandidateAsk{Candidate: candidate, Input: input}
|
||||
if _, err := delivery.EncodeComposeCandidateAsk(ask); err != nil {
|
||||
ask := nodeengine.ComposeCandidateInput{Candidate: candidate, Input: input}
|
||||
if _, err := nec.EncodeAsk(ask); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a delivery.ComposeCandidateAnswer
|
||||
var a nodeengine.ComposeCandidateAnswer
|
||||
if s := shadow.Load(); s == nil {
|
||||
// Unready, never an error the checker reads words of: its agent is starting (ADR 0593 §9).
|
||||
a = delivery.ComposeCandidateAnswer{Node: node, Candidate: candidate, Verdict: delivery.ComposeCandidateUnready,
|
||||
a = nodeengine.ComposeCandidateAnswer{Node: node, Candidate: candidate, Verdict: nec.Unready,
|
||||
Refused: "this node's shadow does not run now (its node agent is starting), so it composes no candidate"}
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), delivery.ComposeAnswerBound)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), nec.AnswerBound)
|
||||
defer cancel()
|
||||
a = s.ComposeCandidate(ctx, ask)
|
||||
}
|
||||
raw, err := delivery.EncodeComposeCandidateAnswer(a)
|
||||
raw, err := nec.EncodeAnswer(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("this node's answer to %s does not fit the seat's shape: %w", candidate, err)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
@@ -58,16 +58,16 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
|
||||
// open, nor after it ended; and a library waiting for its disk is not asked.
|
||||
t.Run("the home engine asks the take once", func(t *testing.T) {
|
||||
var asked []asks.Ask
|
||||
var asked []oc.Ask
|
||||
state := &link.MemState{}
|
||||
h := &link.HandActs{Node: "n1", State: state,
|
||||
JudgeAccess: func(act, access string) (link.Judged, error) {
|
||||
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
|
||||
},
|
||||
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
|
||||
h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||
h.Asker = &occ.Client{Module: occ.EngineAsker("n1"), State: state,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
if a, ok := body.(asks.Ask); ok {
|
||||
if a, ok := body.(oc.Ask); ok {
|
||||
asked = append(asked, a)
|
||||
}
|
||||
return nil
|
||||
@@ -78,7 +78,7 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
|
||||
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
|
||||
}
|
||||
h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()})
|
||||
h.Heard(oc.Warrant{Ask: asked[0].ID, Asker: occ.EngineAsker("n1"), Outcome: occ.OutcomeExpired, At: occ.FormatTime(time.Now())})
|
||||
askTakes(h, waiting, nil)
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
|
||||
|
||||
+14
-1
@@ -936,6 +936,11 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// which the launcher already reads as "run whatever is on disk now".
|
||||
aside, standAside := context.WithCancel(ctx)
|
||||
defer standAside()
|
||||
// **And it ends between applies too** (novox/hq issue 1585): whichever way the link ends, the apply in flight ends and
|
||||
// saves before the process does. Deferred after standAside, so it runs before it: nothing is cancelled under it.
|
||||
defer func() { _ = stopBetweenApplies(say, stopBound) }()
|
||||
// What an earlier stop cut off before its rename is removed, and said, before anything saves.
|
||||
removeLeftoverSaves(opts.state, say)
|
||||
stoodAside := false
|
||||
membership := link.Membership{
|
||||
Node: mine.Node,
|
||||
@@ -1059,6 +1064,14 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
|
||||
trust.onChange(func() { factsOfNow(aside, current.get()) })
|
||||
go followTrust(aside, queue, trust, say)
|
||||
// And the status view's `settled`, for a failed pull from a store to be tried again once the store's holder has settled
|
||||
// (design 51, ENG-11).
|
||||
go followSettled(aside, queue, func() settledTold {
|
||||
if ag := current.get(); ag != nil {
|
||||
return ag
|
||||
}
|
||||
return nil
|
||||
}, say)
|
||||
// And the login it dials as, which the vault's node-logins names for this node (novox/hq ADR 0583 §1).
|
||||
rotator := newRotator(queue, opts.state, opts.timeout, say)
|
||||
go followLogins(aside, queue, mine.Node, func(nl delivery.NodeLogin) {
|
||||
@@ -2019,7 +2032,7 @@ func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say
|
||||
return
|
||||
}
|
||||
say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
|
||||
os.Exit(0)
|
||||
exitBetweenApplies(say)
|
||||
}
|
||||
|
||||
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -132,6 +133,57 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
|
||||
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
||||
// what the other recorded.
|
||||
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
||||
// Issue 1585: the engine's stop waits for the apply in flight to end and save, and then lets no save start; before,
|
||||
// the process ended under it and a save cut before its rename left its temporary file and the state before it.
|
||||
t.Run("the stop waits for the apply in flight and its save", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
applying.Lock()
|
||||
var said []string
|
||||
var mu sync.Mutex
|
||||
waiting := make(chan struct{})
|
||||
say := func(line string) {
|
||||
mu.Lock()
|
||||
said = append(said, line)
|
||||
mu.Unlock()
|
||||
if strings.Contains(line, "waiting up to") {
|
||||
close(waiting)
|
||||
}
|
||||
}
|
||||
stopped := make(chan func(), 1)
|
||||
go func() { stopped <- stopBetweenApplies(say, time.Hour) }()
|
||||
// Said before it blocks, and it cannot end before the apply lets go: so it has not ended now.
|
||||
<-waiting
|
||||
select {
|
||||
case release := <-stopped:
|
||||
release()
|
||||
applying.Unlock()
|
||||
t.Fatal("the stop did not wait for the apply in flight")
|
||||
default:
|
||||
}
|
||||
if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applying.Unlock()
|
||||
release := <-stopped
|
||||
saved := make(chan error, 1)
|
||||
go func() { saved <- store.Save(path, store.State{}) }()
|
||||
// A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look.
|
||||
select {
|
||||
case <-saved:
|
||||
release()
|
||||
t.Fatal("a save started after the stop")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
if err := <-saved; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") {
|
||||
t.Errorf("said %q", said)
|
||||
}
|
||||
})
|
||||
// A host is built for one system at link time, and a test binary has no link time: this asks
|
||||
// the machine it runs on, and stands aside where the answer is no.
|
||||
built, err := system.For("arch")
|
||||
|
||||
@@ -120,7 +120,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
}
|
||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed})
|
||||
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait})
|
||||
}
|
||||
|
||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||
@@ -509,6 +509,12 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
// composes its own (issue 774).
|
||||
Applied: appliedStoreFor(opts.state),
|
||||
Inactive: inactiveIn(&liveness.Exec{Run: liveness.Runner(opts.run()), System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
||||
Held: func() map[string]string {
|
||||
if c := composerNow.Load(); c != nil {
|
||||
return c.LastHeld()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||
if err != nil {
|
||||
@@ -725,12 +731,19 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
|
||||
if got.Refused != "" {
|
||||
return failedWith(got.Refused)
|
||||
}
|
||||
// An image that could not be pulled from a store is said with its holder, for the engine to try the apply again once the
|
||||
// holder has settled (design 51, ENG-8).
|
||||
failedOn := func(why string) agent.Result {
|
||||
res := failedWith(why)
|
||||
res.Pulls = pullsOf(d, got.Err)
|
||||
return res
|
||||
}
|
||||
results := apply.ModuleResults(d, got.Outcome, got.Err)
|
||||
if r, ok := results[name]; ok && r.Failed {
|
||||
return failedWith(r.Detail)
|
||||
return failedOn(r.Detail)
|
||||
}
|
||||
if got.Err != nil {
|
||||
return failedWith(got.Err.Error())
|
||||
return failedOn(got.Err.Error())
|
||||
}
|
||||
if got.Probe != nil {
|
||||
return failedWith(got.Probe.partial().Error())
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -598,12 +599,12 @@ func h8candidate(t *testing.T, b *itBed, fp, running string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := got.(json.RawMessage)
|
||||
a, err := delivery.DecodeComposeCandidateAnswer(raw)
|
||||
a, err := nec.DecodeAnswer(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("the answer %s: %v", raw, err)
|
||||
}
|
||||
if a.Verdict != delivery.ComposeCandidateComposed || len(a.Modules) != 1 || a.Modules[0].Module != "web" ||
|
||||
a.Modules[0].FP != fpIt("c") || !a.Modules[0].Changed || a.Took > delivery.ComposeAnswerBound.Milliseconds() {
|
||||
if a.Verdict != nec.Composed || len(a.Modules) != 1 || a.Modules[0].Module != "web" ||
|
||||
a.Modules[0].Fp != fpIt("c") || !a.Modules[0].Changed || a.Took > nec.AnswerBound.Milliseconds() {
|
||||
t.Errorf("the node's answer to a candidate changing web: %+v", a)
|
||||
}
|
||||
after, _ := b.composed.Status()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
)
|
||||
|
||||
// pullsOf is the images an apply could not find or pull, each with the seat it came through and that seat's holder as the
|
||||
// declaration names them beside the container (design 51 *distribution's first delivery*, ENG-8, ENG-10). An image the
|
||||
// declaration names no holder for (a public registry's) carries none: its pull is not tried again on a store settling.
|
||||
func pullsOf(d *declaration.Declaration, err error) []agent.PullFailure {
|
||||
var out []agent.PullFailure
|
||||
for _, p := range apply.PullErrors(err) {
|
||||
f := agent.PullFailure{Image: p.Image, Started: p.Started}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && c.Image == p.Image && c.ImageHolder != "" {
|
||||
f.Seat, f.Holder = c.ImageSeat, c.ImageHolder
|
||||
break
|
||||
}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// settledEvent is where the status view says a module has settled on a node: its have reached its want, running and healthy
|
||||
// (design 47; the view's `settled`, said once for a module on a node at one composition). The engine hears it for every module
|
||||
// and tells its agent, which tries again, once per settled, a failed apply that waits on that module as the holder of the store
|
||||
// its image comes from (design 51 *distribution's first delivery*, issue 1966 ENG-11). The grants part grants the engine this
|
||||
// subject (NodeAgentHears); a refused subscription is said, and made again when the next link opens.
|
||||
const settledEvent = "mesh.mod.mesh-delivery-view.event.settled"
|
||||
|
||||
// settledBody is what of the event the engine reads: the module that settled, on whichever node.
|
||||
type settledBody struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// settledTold is who is told a holder has settled.
|
||||
type settledTold interface{ Settled(holder string) }
|
||||
|
||||
// followSettled hears the status view's `settled` on every link, never on a timer, until ctx ends. now names the agent running
|
||||
// now (nil while the node agent begins again): a settled heard while there is none is not lost for it, since a new agent applies
|
||||
// again every have that failed at its start.
|
||||
func followSettled(ctx context.Context, session *link.Session, now func() settledTold, say link.Announce) {
|
||||
said := ""
|
||||
for ctx.Err() == nil {
|
||||
linked, cancel, err := session.WhenLinked(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
events, stop, herr := session.Hear(settledEvent)
|
||||
if herr != nil {
|
||||
if line := fmt.Sprintf("a store settling is not heard yet (%s: %v): a failed pull waits for the next link, or for compose", settledEvent, herr); line != said {
|
||||
said = line
|
||||
say(line)
|
||||
}
|
||||
} else {
|
||||
said = ""
|
||||
listen(linked, events, now, say)
|
||||
stop()
|
||||
}
|
||||
<-linked.Done()
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// listen tells the agent each settled it hears, until the link ends.
|
||||
func listen(linked context.Context, events <-chan []byte, now func() settledTold, say link.Announce) {
|
||||
for {
|
||||
select {
|
||||
case <-linked.Done():
|
||||
return
|
||||
case raw, ok := <-events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var b settledBody
|
||||
if json.Unmarshal(raw, &b) != nil || b.Module == "" {
|
||||
continue
|
||||
}
|
||||
if told := now(); told != nil {
|
||||
told.Settled(b.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// stopBound is how long a stopping engine waits for the apply in flight to end and save what it did: under the service
|
||||
// manager's own stop bound (90 s by default, then every process of the unit is killed), so the wait ends with a line in
|
||||
// the journal rather than with a kill.
|
||||
const stopBound = 60 * time.Second
|
||||
|
||||
// stopBetweenApplies is how this engine ends, whichever way (a stop of the service, standing aside for a successor, a
|
||||
// move to another bus): it waits, up to bound, for the apply in flight to end, then lets no save start and waits for
|
||||
// the one writing (novox/hq issue 1585). An apply runs detached from the engine's own stop (applyContext), so without
|
||||
// this the process ended under it: what it unpacked was never recorded, and a save cut between its write and its
|
||||
// rename left a temporary file and the state before it. A stopping engine never gives back what it takes (the process
|
||||
// ends); the release it returns is for a test, and gives back the apply only when this took it.
|
||||
func stopBetweenApplies(say func(string), bound time.Duration) (release func()) {
|
||||
took := true
|
||||
if !applying.TryLock() {
|
||||
say(fmt.Sprintf("stopping: waiting up to %s for the apply in flight to end and save what it did", bound))
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
applying.Lock()
|
||||
close(got)
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
say("stopping: the apply in flight ended and its state is saved")
|
||||
case <-time.After(bound):
|
||||
took = false
|
||||
say(fmt.Sprintf("stopping while an apply still runs after %s: what it did since its last save is not "+
|
||||
"recorded, and the next start applies its module again", bound))
|
||||
}
|
||||
}
|
||||
saves := store.Quiesce()
|
||||
return func() {
|
||||
saves()
|
||||
if took {
|
||||
applying.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// removeLeftoverSaves removes, at the engine's start, the temporary files of saves of the node's state that never
|
||||
// reached their rename, and says how many (issue 1585: about 260 on novox, ~80 MB).
|
||||
func removeLeftoverSaves(statePath string, say func(string)) {
|
||||
removed, err := store.RemoveLeftovers(statePath)
|
||||
if len(removed) > 0 {
|
||||
shown := removed
|
||||
if len(shown) > 5 {
|
||||
shown = append(append([]string(nil), shown[:5]...), "…")
|
||||
}
|
||||
say(fmt.Sprintf("removed %d leftover save(s) of this node's state, cut off before their rename by an earlier "+
|
||||
"stop: %s", len(removed), strings.Join(shown, ", ")))
|
||||
}
|
||||
if err != nil {
|
||||
say(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// exitBetweenApplies ends the process cleanly once no apply runs and no save writes: an exit that is not runLink's
|
||||
// return (a move to another bus). A variable, so a test sees it called rather than the test process ending.
|
||||
var exitBetweenApplies = func(say func(string)) {
|
||||
_ = stopBetweenApplies(say, stopBound)
|
||||
os.Exit(0)
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshvault"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
@@ -259,11 +260,11 @@ func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
|
||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||
return fmt.Errorf("this node's serving key is not an ed25519 key")
|
||||
}
|
||||
raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
|
||||
raw, err := ask(ctx, meshvault.VerbSign, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var a delivery.SignAnswer
|
||||
var a meshvault.SignAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the vault's answer is not a signing: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,20 +3,27 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103155154-cf5812101b0b
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 // indirect
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshvault v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,15 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103155154-cf5812101b0b h1:UPkJqQj+1y6WraOxDidkrokNjA25jxmCk+zIRkujjMc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103155154-cf5812101b0b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639 h1:Zt7CfWt2S2W/fVgWNyW2u2+xwz051LXvGxVTNIyfhdM=
|
||||
git.novox.be/novox/mesh-contracts/go v0.0.0-20261104174344-ae5266e77639/go.mod h1:0dD1XEJBJ0G5UYDQxSZI84nuujb6m3tOpJl4C8rQVvg=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639 h1:3FTxvHrv3Hv+ZUbp/ezTrndqdgiS+Z6EUG2XGYJNJr8=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:/V/1wfyQwjE9VbPLsFtkf6uX+SHKUXX34s5JscDuGAs=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639 h1:19PhtrCOwia711Oqdyafvs5EKTVmmgk5Gdb4leCkq3w=
|
||||
git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient v0.0.0-20261104174344-ae5266e77639/go.mod h1:Dj6fbL8Evgy72EWO3Gl3L3Ra8N/nJSzgEVwSGmeRam0=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshvault v0.0.0-20261104174344-ae5266e77639 h1:LyN9de63PwLy2VL0RZWjSsoRf0kSX5zisMpeDvKl42k=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshvault v0.0.0-20261104174344-ae5266e77639/go.mod h1:8dL18Gj/VOzNIBSJGlAdFHc9XonCcsZWm/JcjcpwwbU=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639 h1:YQ6eJvc+RYUVIVf+oX+R4TC1dYL8TLLMZ0/JPDU1tp4=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-20261104174344-ae5266e77639/go.mod h1:TB20TsQuOd5aer+cXL6PbaYyg2RvgL1R4MzD61Oks3A=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639 h1:K/1yIkqAWKGtLEtJ0PibijIdpo5YATL/9IUVXFoEglM=
|
||||
git.novox.be/novox/mesh-contracts/go/seats/operatorchannel v0.0.0-20261104174344-ae5266e77639/go.mod h1:osh0jQyNqYDM27tbxQo6HeUEXgLq9XUewKzNhUXadog=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f h1:jDsfDMd48MXY9Zershc3eRxnhmx2sSpUfQkLq5jAIds=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f/go.mod h1:08OomDXdwVN8/87nyDtv5R3rH/6LoP8SFFXz30bTYCM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -23,5 +35,5 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
+245
-15
@@ -21,6 +21,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -68,6 +69,9 @@ type Have struct {
|
||||
// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
|
||||
// its own slot alone (issue 1126, C1).
|
||||
Sealed delivery.Seals
|
||||
// PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966
|
||||
// ENG-9, delivery.Have.PullWait): read through the have, as node_haves shows it.
|
||||
PullWait *delivery.PullWait
|
||||
}
|
||||
|
||||
// Haves is the node's have bucket as the agent uses it.
|
||||
@@ -82,6 +86,9 @@ type Haves interface {
|
||||
|
||||
// Result is how one apply or removal came out.
|
||||
type Result struct {
|
||||
// Pulls are the images the apply could not find or pull, each with the seat it came through and that seat's holder
|
||||
// where the declaration names them (a store's image, ENG-10): what a failed apply waits on (ENG-8).
|
||||
Pulls []PullFailure
|
||||
Failed bool
|
||||
// Transient says the failure was the bus not answering for a moment (no responders, a timeout, a reconnect, a
|
||||
// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
|
||||
@@ -89,6 +96,14 @@ type Result struct {
|
||||
Detail string
|
||||
}
|
||||
|
||||
// PullFailure is one image an apply could not find or pull. Holder is the module holding the seat the image came through,
|
||||
// "" for an image no store holds (a public registry's): its pull is tried again only by the changes that always start an
|
||||
// apply. Started is when the pull began.
|
||||
type PullFailure struct {
|
||||
Image, Seat, Holder string
|
||||
Started time.Time
|
||||
}
|
||||
|
||||
// Backup is the answer of the backup asked before a module moves.
|
||||
// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
|
||||
// the next start backs the module up again rather than holding it for the operator.
|
||||
@@ -136,6 +151,9 @@ type Env struct {
|
||||
// Revisions are a composition key's revisions the bucket keeps, oldest first; nil reads none. The composition a want
|
||||
// names is taken from them when a newer one replaced it under the same key (ADR 0539 clause 1).
|
||||
Revisions func(ctx context.Context, key string) ([]Update, error)
|
||||
// Held is what the node's composer held in its last pass, each module with why, as the verb composition shows it
|
||||
// for the node's own (issue 774); nil says nothing.
|
||||
Held func() map[string]string
|
||||
// Settled is told a module's apply or removal that succeeded, once what runs holds it (Everything), outside the
|
||||
// agent's lock: the declaration the node applies now names what that composition names, its filled files among
|
||||
// them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host
|
||||
@@ -147,6 +165,9 @@ type Env struct {
|
||||
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
|
||||
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
|
||||
RunsChanged func(why string)
|
||||
// Now is the clock the agent compares a pull's start with the time it heard a store's holder settled by; nil is
|
||||
// time.Now.
|
||||
Now func() time.Time
|
||||
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
|
||||
// transient failure only on RetryTransient or a change of either key.
|
||||
Backoff func(try int) time.Duration
|
||||
@@ -229,11 +250,21 @@ type Agent struct {
|
||||
failedAgain chan string
|
||||
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
||||
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
||||
tries map[string]int
|
||||
gen uint64
|
||||
again chan againAsk
|
||||
transient chan struct{}
|
||||
stopped chan struct{}
|
||||
tries map[string]int
|
||||
// pulls is, for each module whose last apply failed on a pull from a store, what it waits on (ENG-8); settledAt and
|
||||
// settledSeq are, by the module holding a store, when its settled was last heard since this agent started and how many
|
||||
// have been. Under mu.
|
||||
pulls map[string]*pullWait
|
||||
settledAt map[string]time.Time
|
||||
settledSeq map[string]int
|
||||
gen uint64
|
||||
again chan againAsk
|
||||
transient chan struct{}
|
||||
stopped chan struct{}
|
||||
// quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied.
|
||||
quiet chan chan struct{}
|
||||
// armed counts the backoff waits running: the loop is not idle while one may still ask an apply again.
|
||||
armed atomic.Int32
|
||||
}
|
||||
|
||||
type againAsk struct {
|
||||
@@ -273,8 +304,9 @@ func New(env Env) *Agent {
|
||||
}
|
||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
|
||||
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
||||
transient: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})}
|
||||
}
|
||||
|
||||
// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition
|
||||
@@ -328,6 +360,19 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
dirty[m] = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
case done := <-a.quiet:
|
||||
// Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest.
|
||||
if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(wants)+len(comps) == 0 &&
|
||||
a.armed.Load() == 0 {
|
||||
close(done)
|
||||
} else {
|
||||
go func() {
|
||||
select {
|
||||
case a.quiet <- done:
|
||||
case <-a.stopped:
|
||||
}
|
||||
}()
|
||||
}
|
||||
case ask := <-a.again:
|
||||
// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
|
||||
a.mu.Lock()
|
||||
@@ -386,6 +431,27 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// waitIdle returns once the loop has taken in and applied every change sent to it before the call (a want or composition
|
||||
// put, a settled heard, a retry asked), with no backoff wait still running, so a caller can count what was applied with no clock; false when ctx ends first.
|
||||
func (a *Agent) waitIdle(ctx context.Context) bool {
|
||||
done := make(chan struct{})
|
||||
select {
|
||||
case a.quiet <- done:
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-a.stopped:
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-a.stopped:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Retry asks for the modules to be applied again from the keys they have, though nothing changed: what ADR 0360 §11
|
||||
// needs for a container that waits because it said busy. It never blocks; a retry asked while eight wait is dropped, and
|
||||
// asked again by its caller's next look.
|
||||
@@ -1302,20 +1368,35 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
|
||||
a.mu.Unlock()
|
||||
}
|
||||
a.mu.Lock()
|
||||
if r, ok := a.running[module]; ok && r.FP == w.Want.FP && r.Composed == w.Want.Composed && r.Phase != delivery.PhaseRemoved {
|
||||
r, runs := a.running[module]
|
||||
runsWant := runs && r.FP == w.Want.FP && r.Composed == w.Want.Composed && r.Phase != delivery.PhaseRemoved
|
||||
att, tried := a.attempt[module]
|
||||
same := tried && att.FP == w.Want.FP && att.Composed == w.Want.Composed
|
||||
if same && att.Held {
|
||||
// Held for a failed backup: looked at before what runs, since a hold may name what the want names (its have kept
|
||||
// the composition it held, nats on novox, 2026-11-04), and an answer in a later revision of the want would then
|
||||
// be swallowed by "it runs what its want names" (issue 1585, the second failure). The operator's answer, again or
|
||||
// skip, in a revision after the one the hold took, is acted on whatever runs; any other new revision of the want
|
||||
// is the answer only for a move (ADR 0568 §4), as before.
|
||||
answered := w.Rev > att.AnsweredRev && (w.Want.Backup == delivery.BackupAgain || w.Want.Backup == delivery.BackupSkip)
|
||||
if !answered && (runsWant || (att.AnsweredRev >= w.Rev && att.CompRev == comp.Rev)) {
|
||||
a.sayOnce("held:"+module, fmt.Sprintf("%s is held for a failed backup before %s: it waits for the operator's "+
|
||||
"answer, and its want (revision %d) carries none since the hold (revision %d)", module, w.Want.FP, w.Rev, att.AnsweredRev))
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if answered {
|
||||
a.sayOnce("answered:"+module+":"+fmt.Sprint(w.Rev), fmt.Sprintf("%s is held for a failed backup, and "+
|
||||
"its want (revision %d) carries the operator's answer %q: acted on now", module, w.Rev, w.Want.Backup))
|
||||
}
|
||||
} else if runsWant {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
att, tried := a.attempt[module]
|
||||
same := tried && att.FP == w.Want.FP && att.Composed == w.Want.Composed
|
||||
if same && !att.Held && att.WantRev == w.Rev && att.CompRev == comp.Rev {
|
||||
a.mu.Unlock()
|
||||
return // failed already for exactly these keys: retried by the next change of either
|
||||
}
|
||||
if same && att.Held && att.AnsweredRev >= w.Rev && att.CompRev == comp.Rev {
|
||||
a.mu.Unlock()
|
||||
return // held for a failed backup, and no answer since
|
||||
}
|
||||
if module == NodeModule {
|
||||
// Not while a module's composition cannot be read or is refused: what it carries may be what left the node's own,
|
||||
// and the node's own apply would remove it, held out by nothing (issue 1945, the review's gap 2).
|
||||
@@ -1372,10 +1453,27 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
a.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
if h.Phase == string(delivery.PhaseHeld) && (w.Want.Backup == delivery.BackupAgain || w.Want.Backup == delivery.BackupSkip) {
|
||||
// Held for a failed backup, and the want already carries the operator's answer, written before this engine
|
||||
// started (nats on novox, 2026-11-04: "Back up again" answered, then the engine restarted and took the
|
||||
// answer for the hold's own revision, so it was never acted on; issue 1585). The answer is acted on now:
|
||||
// again backs it up and applies (a.prior makes the apply back it up first), skip applies without one.
|
||||
// An answer the engine already acted on before its stop, and whose backup failed once more, is so acted
|
||||
// on once more at this start: one backup more, never one skipped.
|
||||
a.mu.Lock()
|
||||
a.sayOnce("answered:"+module, fmt.Sprintf("%s is held for a failed backup, and its want carries the operator's "+
|
||||
"answer %q: acted on at this start", module, w.Want.Backup))
|
||||
a.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
if h.Phase == string(delivery.PhaseHeld) {
|
||||
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
|
||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so
|
||||
// why it holds is read from the journal and not guessed (issue 1585: this branch was silent).
|
||||
a.mu.Lock()
|
||||
a.sayOnce("held:"+module, fmt.Sprintf("%s is held for a failed backup (%s) at this start: it keeps what "+
|
||||
"its have names, %s, and waits for the operator's answer; its want (revision %d) carries none",
|
||||
module, h.Detail, short(h.Composed), w.Rev))
|
||||
a.attempt[module] = attempt{FP: w.Want.FP, Composed: w.Want.Composed, WantRev: w.Rev, CompRev: comp.Rev, Held: true,
|
||||
AnsweredRev: w.Rev}
|
||||
if h.Composed != "" {
|
||||
@@ -1494,6 +1592,9 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
|
||||
res := a.env.Apply(ctx, comp.C, others)
|
||||
a.mu.Lock()
|
||||
if !res.Failed || res.Transient {
|
||||
delete(a.pulls, module)
|
||||
}
|
||||
what := fmt.Sprintf("%s at %s", module, fp)
|
||||
if module == NodeModule {
|
||||
what = "the node's own composition"
|
||||
@@ -1509,7 +1610,9 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
if a.env.Backoff != nil {
|
||||
wait := a.env.Backoff(try)
|
||||
when = "in " + wait.String()
|
||||
a.armed.Add(1)
|
||||
time.AfterFunc(wait, func() {
|
||||
defer a.armed.Add(-1)
|
||||
select {
|
||||
case a.again <- againAsk{module, gen}:
|
||||
case <-a.stopped:
|
||||
@@ -1528,11 +1631,22 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
delete(a.tries, module)
|
||||
if res.Failed {
|
||||
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
|
||||
detail := res.Detail
|
||||
again := a.notePullFailed(module, res.Pulls)
|
||||
if words := a.pullWords(module); words != "" {
|
||||
detail += words
|
||||
}
|
||||
wait := a.pullWaitOf(module)
|
||||
a.mu.Unlock()
|
||||
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: detail,
|
||||
PullWait: wait})
|
||||
if again != "" {
|
||||
a.askPullAgain(module, again)
|
||||
}
|
||||
return
|
||||
}
|
||||
delete(a.pulls, module)
|
||||
delete(a.attempt, module)
|
||||
delete(a.notComposed, module) // the apply says what it runs now
|
||||
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
|
||||
@@ -1568,6 +1682,7 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
|
||||
return
|
||||
}
|
||||
delete(a.attempt, module)
|
||||
delete(a.pulls, module)
|
||||
delete(a.had, module)
|
||||
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
|
||||
a.mu.Unlock()
|
||||
@@ -1778,3 +1893,118 @@ func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pullWait is what a module whose apply failed on a pull from a store waits on: the store's holder settling (ENG-8,
|
||||
// design 51 *distribution's first delivery*).
|
||||
type pullWait struct {
|
||||
PullFailure
|
||||
// retriedFor is the count of the holder's settled the last retry answered: a settled answers once.
|
||||
retriedFor int
|
||||
// last is what triggered the last retry, in words, and lastAt when; the have and the journal say it.
|
||||
last string
|
||||
lastAt time.Time
|
||||
}
|
||||
|
||||
func (a *Agent) now() time.Time {
|
||||
if a.env.Now != nil {
|
||||
return a.env.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// notePullFailed records the pull a failed apply met from a store (the first image with a holder), keeping the settled it was
|
||||
// last tried again for. It answers, when the holder's settled was heard after the pull started and not yet answered, the
|
||||
// words of the retry to make at once (the race just before `settled`: ENG-11), else "". a.mu is held.
|
||||
func (a *Agent) notePullFailed(module string, pulls []PullFailure) string {
|
||||
var found *PullFailure
|
||||
for i := range pulls {
|
||||
if pulls[i].Holder != "" {
|
||||
found = &pulls[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
delete(a.pulls, module)
|
||||
return ""
|
||||
}
|
||||
pw := &pullWait{PullFailure: *found}
|
||||
if before := a.pulls[module]; before != nil {
|
||||
pw.retriedFor, pw.last, pw.lastAt = before.retriedFor, before.last, before.lastAt
|
||||
}
|
||||
a.pulls[module] = pw
|
||||
holder := found.Holder
|
||||
if seq := a.settledSeq[holder]; seq > pw.retriedFor && a.settledAt[holder].After(found.Started) {
|
||||
return a.markPullRetry(pw, seq, fmt.Sprintf("%s settled at %s, after the pull of %s began at %s", holder,
|
||||
a.settledAt[holder].Format("15:04:05"), found.Image, found.Started.Format("15:04:05")))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// markPullRetry notes that pw is tried again for the holder's settled number seq, and answers the words that say so. a.mu
|
||||
// is held.
|
||||
func (a *Agent) markPullRetry(pw *pullWait, seq int, why string) string {
|
||||
pw.retriedFor, pw.last, pw.lastAt = seq, why, a.now()
|
||||
return why
|
||||
}
|
||||
|
||||
// pullWords is what a failed have says of the wait: whose settling it is tried again on and what triggered the last retry.
|
||||
// "" for a failure that waits on no holder. a.mu is held.
|
||||
func (a *Agent) pullWords(module string) string {
|
||||
pw := a.pulls[module]
|
||||
if pw == nil {
|
||||
return ""
|
||||
}
|
||||
words := fmt.Sprintf("; it is tried again once %s has settled", pw.Holder)
|
||||
if pw.last != "" {
|
||||
words += "; last tried again at " + pw.lastAt.Format("15:04:05") + ": " + pw.last
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
// askPullAgain asks the apply of a module again, because why, as a person's verb compose does (RetryFailed): said in the
|
||||
// journal, never blocking.
|
||||
func (a *Agent) askPullAgain(module, why string) {
|
||||
select {
|
||||
case a.failedAgain <- module:
|
||||
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store; applied again because %s", module, why))
|
||||
default:
|
||||
a.env.Say(fmt.Sprintf("%s: its image could not be pulled from the store, and %s, but the engine has too many retries "+
|
||||
"asked; ask it with compose", module, why))
|
||||
}
|
||||
}
|
||||
|
||||
// Settled is the status view's word that a module has settled on some node (ENG-11): the engine's hearing tells it for every
|
||||
// module it hears. When the module holds a store a failed apply of this node waits on, that apply is tried again once for
|
||||
// this settled; a settled of any other module starts nothing. A pull that began after this settled, or has failed before it
|
||||
// and is answered, is not tried again for it (notePullFailed).
|
||||
func (a *Agent) Settled(holder string) {
|
||||
a.mu.Lock()
|
||||
a.settledSeq[holder]++
|
||||
a.settledAt[holder] = a.now()
|
||||
seq := a.settledSeq[holder]
|
||||
var asks []string
|
||||
var modules []string
|
||||
for module, pw := range a.pulls {
|
||||
att, failed := a.attempt[module]
|
||||
if pw.Holder != holder || !failed || att.Held || att.Transient || pw.retriedFor >= seq {
|
||||
continue
|
||||
}
|
||||
modules = append(modules, module)
|
||||
asks = append(asks, a.markPullRetry(pw, seq, fmt.Sprintf("%s settled", holder)))
|
||||
}
|
||||
a.mu.Unlock()
|
||||
for i, module := range modules {
|
||||
a.askPullAgain(module, asks[i])
|
||||
}
|
||||
}
|
||||
|
||||
// pullWaitOf is the pull a failed apply of module waits on, as its have carries it (ENG-9): the image, the seat and its
|
||||
// holder, when the pull began, and the last retry with what triggered it; nil when it waits on no holder. a.mu is held.
|
||||
func (a *Agent) pullWaitOf(module string) *delivery.PullWait {
|
||||
pw := a.pulls[module]
|
||||
if pw == nil {
|
||||
return nil
|
||||
}
|
||||
return &delivery.PullWait{Image: pw.Image, Seat: pw.Seat, Holder: pw.Holder, Began: pw.Started, Retried: pw.last,
|
||||
RetriedAt: pw.lastAt}
|
||||
}
|
||||
|
||||
@@ -8,10 +8,12 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -114,10 +116,14 @@ type bed struct {
|
||||
// transient marks failWith as the bus not answering; failTimes > 0 fails only the first that many applies.
|
||||
transient bool
|
||||
failTimes int
|
||||
backup func(module string) Backup
|
||||
kept map[string][]json.RawMessage
|
||||
others [][]delivery.Composed
|
||||
said []string
|
||||
// pullsFn, when set, is the images a failed apply could not pull (called as the apply fails); during runs at the start of
|
||||
// each apply, before it answers.
|
||||
pullsFn func() []PullFailure
|
||||
during func()
|
||||
backup func(module string) Backup
|
||||
kept map[string][]json.RawMessage
|
||||
others [][]delivery.Composed
|
||||
said []string
|
||||
// settled is each module Settled was told, and settledRan whether what runs held it then.
|
||||
settled []string
|
||||
settledRan []bool
|
||||
@@ -129,6 +135,9 @@ func newBed(t *testing.T) *bed {
|
||||
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
|
||||
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
|
||||
Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
|
||||
if b.during != nil {
|
||||
b.during()
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.others = append(b.others, others)
|
||||
@@ -138,7 +147,11 @@ func newBed(t *testing.T) *bed {
|
||||
}
|
||||
b.applied = append(b.applied, name)
|
||||
if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
|
||||
return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
|
||||
res := Result{Failed: true, Transient: b.transient, Detail: b.failWith}
|
||||
if b.pullsFn != nil {
|
||||
res.Pulls = b.pullsFn()
|
||||
}
|
||||
return res
|
||||
}
|
||||
return Result{}
|
||||
},
|
||||
@@ -180,6 +193,17 @@ func newBed(t *testing.T) *bed {
|
||||
return b
|
||||
}
|
||||
|
||||
// idle waits until the agent has taken in and applied everything sent to it so far: what is counted after it is all there
|
||||
// will be, with no clock.
|
||||
func (b *bed) idle() {
|
||||
b.t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if !b.ag.waitIdle(ctx) {
|
||||
b.t.Fatalf("the agent's loop did not come to rest")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bed) run() (stop func()) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
@@ -460,6 +484,115 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
||||
t.Errorf("a module running its want was applied again (%d applies)", n)
|
||||
}
|
||||
})
|
||||
// design 51 (ENG-8, ENG-11): a pull from a store that failed is tried again when the store's holder settles, once per
|
||||
// settled, never on a clock; the have says whom it waits on and what triggered the last retry.
|
||||
t.Run("a pull from a store waits for its holder to settle", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
var clock atomic.Int64
|
||||
now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
|
||||
tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
|
||||
b.env.Now = now
|
||||
b.env.Backoff = func(int) time.Duration { return time.Millisecond }
|
||||
b.failWith = "pulling image reg/x@sha256:1: manifest unknown"
|
||||
b.pullsFn = func() []PullFailure {
|
||||
return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: tick()},
|
||||
{Image: "public/y@sha256:2"}}
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.put(ck, craw)
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "the have does not say whom it waits on", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "tried again once distribution has settled")
|
||||
})
|
||||
b.idle()
|
||||
if n := len(b.appliedList()); n != 1 {
|
||||
t.Fatalf("a failed pull was tried again by the clock (%d applies)", n)
|
||||
}
|
||||
// A settled of another module starts nothing.
|
||||
b.ag.Settled("gitea")
|
||||
b.idle()
|
||||
if n := len(b.appliedList()); n != 1 {
|
||||
t.Fatalf("a settled of another module applied the module (%d applies)", n)
|
||||
}
|
||||
// The holder's settled: tried again once, whose pull fails again after it (started later): failed again, no more.
|
||||
tick()
|
||||
b.ag.Settled("distribution")
|
||||
eventually(t, "not tried again on the holder's settled", func() bool { return len(b.appliedList()) == 2 })
|
||||
eventually(t, "the have does not say what triggered the retry", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.Phase == string(delivery.PhaseFailed) && strings.Contains(h.Detail, "last tried again at") && strings.Contains(h.Detail, "distribution settled")
|
||||
})
|
||||
b.idle()
|
||||
if n := len(b.appliedList()); n != 2 {
|
||||
t.Fatalf("a second failure after the same settled started a further apply (%d applies)", n)
|
||||
}
|
||||
// The failed have carries the pull it waits on (ENG-9, read through node_haves): the image, whose settled it waits
|
||||
// on, and what triggered the last retry.
|
||||
if w := b.haves.last("web").PullWait; w == nil || w.Holder != "distribution" || w.Image != "reg/x@sha256:1" ||
|
||||
w.Began.IsZero() || !strings.Contains(w.Retried, "distribution settled") || w.RetriedAt.IsZero() {
|
||||
t.Fatalf("the failed have says it waits on %+v", w)
|
||||
}
|
||||
// The next settled answers once more, and the pull works: the module runs, and waits on nothing.
|
||||
b.mu.Lock()
|
||||
b.failWith = ""
|
||||
b.mu.Unlock()
|
||||
tick()
|
||||
b.ag.Settled("distribution")
|
||||
eventually(t, "not tried again on the next settled", func() bool { return len(b.appliedList()) == 3 })
|
||||
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
||||
if w := b.haves.last("web").PullWait; w != nil {
|
||||
t.Fatalf("a running module's have still waits on %+v", w)
|
||||
}
|
||||
b.ag.Settled("distribution")
|
||||
b.idle()
|
||||
if n := len(b.appliedList()); n != 3 {
|
||||
t.Fatalf("a running module was applied again by a settled (%d applies)", n)
|
||||
}
|
||||
})
|
||||
// The race just before `settled`: the holder's settled is heard while the pull is in flight, and the pull fails after it;
|
||||
// applied again at once, once, and a settled heard before the pull began applies nothing at once.
|
||||
t.Run("a settled heard while the pull ran is not missed", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
var clock atomic.Int64
|
||||
now := func() time.Time { return time.Unix(1_000_000, clock.Load()) }
|
||||
tick := func() time.Time { clock.Add(int64(time.Second)); return now() }
|
||||
b.env.Now = now
|
||||
b.failWith = "pulling image reg/x@sha256:1: connection refused"
|
||||
began := tick()
|
||||
first := true
|
||||
b.during = func() {
|
||||
if first {
|
||||
first = false
|
||||
tick()
|
||||
b.ag.Settled("distribution") // the holder settles while this apply's pull is in flight
|
||||
}
|
||||
}
|
||||
b.pullsFn = func() []PullFailure {
|
||||
started := began
|
||||
if len(b.applied) > 1 {
|
||||
started = tick() // the retry's pull begins after the settled
|
||||
}
|
||||
return []PullFailure{{Image: "reg/x@sha256:1", Seat: "mesh-artifact-store", Holder: "distribution", Started: started}}
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.put(ck, craw)
|
||||
b.wants.put(wk, wraw)
|
||||
eventually(t, "the settled heard during the pull was missed", func() bool { return len(b.appliedList()) == 2 })
|
||||
b.idle()
|
||||
if n := len(b.appliedList()); n != 2 {
|
||||
t.Fatalf("the race retry failing again started a further apply (%d applies)", n)
|
||||
}
|
||||
})
|
||||
// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
|
||||
// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
|
||||
// retrying meanwhile, never failed.
|
||||
@@ -594,7 +727,8 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
||||
}
|
||||
|
||||
// At a start: a backup the node agent's own stop interrupted (postgres on novox, 2026-10-31) is no failure, so it is
|
||||
// backed up again and applied while the want asks a backup; a backup that failed stays held for the operator.
|
||||
// backed up again and applied while the want asks a backup; a backup that failed stays held for the operator's
|
||||
// answer, and an answer the want already carries is acted on.
|
||||
for _, row := range []struct {
|
||||
detail, backup string
|
||||
backups, applied int
|
||||
@@ -604,7 +738,11 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
||||
{BackupInterrupted, delivery.BackupAgain, 1, 1, delivery.PhaseRunning, false},
|
||||
// novox's postgres: its held have, written before mesh-host #189, names nothing it runs.
|
||||
{BackupInterrupted, "", 1, 1, delivery.PhaseRunning, true},
|
||||
{"restic: the repository is locked", delivery.BackupAgain, 0, 0, delivery.PhaseHeld, false},
|
||||
// A backup that failed, and the want carries the operator's answer, written before this start (nats on novox,
|
||||
// 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold.
|
||||
{"restic: the repository is locked", delivery.BackupAgain, 1, 1, delivery.PhaseRunning, false},
|
||||
{"restic: the repository is locked", delivery.BackupSkip, 0, 1, delivery.PhaseRunning, false},
|
||||
{"restic: the repository is locked", "", 0, 0, delivery.PhaseHeld, false},
|
||||
} {
|
||||
r := newBed(t)
|
||||
r.comps.before(ck1, craw1)
|
||||
@@ -630,6 +768,78 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
||||
}
|
||||
stopR()
|
||||
}
|
||||
|
||||
// The live shape (nats on novox, 2026-11-04, issue 1585, the second failure): a module the node composes itself, of
|
||||
// policy record, its want Self with the yes for its composition, and its held have naming that very composition (what
|
||||
// runs is what the want names). The operator's answer acted on at the start, and in a revision of the want that
|
||||
// arrives while the module is held; the engine's read tool says the want's answer and the hold.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
atStart string // the answer the want carries when the engine starts
|
||||
later string // the answer a later revision carries; "" writes none
|
||||
backups int
|
||||
holdsTill bool // it is held, and said held, before the later revision
|
||||
}{
|
||||
{"answered before the start", delivery.BackupAgain, "", 1, false},
|
||||
{"answered while held", "", delivery.BackupAgain, 1, true},
|
||||
{"skipped while held", "", delivery.BackupSkip, 0, true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
r := newBed(t)
|
||||
r.env.SelfComposed, r.kept = true, nil
|
||||
r.env.Policy = func(context.Context, string) string { return catalogue.PolicyRecord }
|
||||
ck, craw, dig := composition(t, "nats", fp2, true, file("nats", "x", "2"))
|
||||
r.comps.before(ck, craw)
|
||||
wk, _ := delivery.WantKey(node, "nats")
|
||||
selfWant := func(backup string) []byte {
|
||||
raw, err := delivery.EncodeWant(delivery.Want{FP: fp2, Self: true, Yes: dig, Backup: backup})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
r.wants.before(wk, selfWant(row.atStart))
|
||||
r.haves.stored = map[string]Have{"nats": {Module: "nats", FP: fp2, Composed: dig, Phase: string(delivery.PhaseHeld),
|
||||
Detail: "the backup failed: sh: exit status 1"}}
|
||||
stopR := r.run()
|
||||
defer stopR()
|
||||
if row.holdsTill {
|
||||
eventually(t, "the hold at the start is not said", func() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, s := range r.said {
|
||||
if strings.Contains(s, "nats is held for a failed backup") && strings.Contains(s, "carries none") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
view, err := r.ag.Composition(context.Background(), "nats", "", false)
|
||||
if err != nil || view.Attempt == nil || !view.Attempt.Held || view.Want == nil || view.Want.Backup != "" || !view.Started {
|
||||
t.Fatalf("the read tool does not say the hold: %+v %+v %v", view.Attempt, view.Want, err)
|
||||
}
|
||||
if got := r.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("applied behind a hold with no answer: %v", got)
|
||||
}
|
||||
r.wants.put(wk, selfWant(row.later))
|
||||
eventually(t, "the read tool does not say the answer", func() bool {
|
||||
v, err := r.ag.Composition(context.Background(), "nats", "", false)
|
||||
return err == nil && v.Want != nil && v.Want.Backup == row.later && v.Want.Revision > view.Want.Revision
|
||||
})
|
||||
}
|
||||
eventually(t, "the answer was not acted on", func() bool { return len(r.appliedList()) == 1 })
|
||||
eventually(t, "not running after the answer", func() bool {
|
||||
h := r.haves.last("nats")
|
||||
return h.Phase == string(delivery.PhaseRunning) && h.Composed == dig
|
||||
})
|
||||
r.mu.Lock()
|
||||
n := len(r.backups)
|
||||
r.mu.Unlock()
|
||||
if n != row.backups {
|
||||
t.Errorf("%d backups for the answer, want %d", n, row.backups)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
||||
@@ -1312,6 +1522,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
defer historyMu.Unlock()
|
||||
history[key] = append(history[key], Update{Key: key, Value: raw, Revision: uint64(len(history[key]) + 1)})
|
||||
}
|
||||
b.env.Held = func() map[string]string { return map[string]string{"mesh-build-go": "its binding is unset"} }
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
||||
@@ -1493,6 +1704,16 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
if _, err := b.ag.Composition(context.Background(), "db", "sha256:"+strings.Repeat("0", 64), false); err == nil {
|
||||
t.Error("a digest no revision has was answered")
|
||||
}
|
||||
// Issue 774 (ace, 2026-11-04 01:29): the node's own revision shows its own fields, the provisioner client's account
|
||||
// and the adoption among them, and what the composer's last pass held.
|
||||
ownKey, ownRaw, ownDig := encodedAs(t, delivery.Composed{Node: node, Own: delivery.NodeOwn{Provisioner: "mesh-provisioner-client",
|
||||
Adoption: &declaration.Adoption{Taken: []string{"web"}}}}, func() string { k, _ := delivery.ComposedNodeKey(node); return k }())
|
||||
keep(ownKey, ownRaw)
|
||||
if view, err := b.ag.Composition(context.Background(), NodeModule, ownDig, false); err != nil || view.Own == nil ||
|
||||
view.Own.Provisioner != "mesh-provisioner-client" || view.Own.Adoption == nil || view.Own.Adoption.Taken[0] != "web" ||
|
||||
view.Own.Held["mesh-build-go"] == "" {
|
||||
t.Errorf("the node's own composition reads %+v %+v (%v)", view, view.Own, err)
|
||||
}
|
||||
// novox/hq ADR 0593 §6: the engine publishes each composition with its plan in its composition state; the verb reads
|
||||
// that revision by the digest its have and want name, its resources' (the plan left out of the digest).
|
||||
planned, _ := delivery.DecodeComposed(draw)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/judge"
|
||||
)
|
||||
@@ -20,6 +21,13 @@ type CompositionView struct {
|
||||
Module string `json:"module"`
|
||||
// Want is the module's want as the agent holds it; nil when it holds none.
|
||||
Want *WantView `json:"want,omitempty"`
|
||||
// Attempt is what the agent holds of the module's last apply that did not end running: failed, failed for a moment
|
||||
// of the bus, or held for a failed backup, with the want's revision that hold took as answered; nil when none
|
||||
// (issue 1585: why a module stays held was not readable from the engine).
|
||||
Attempt *AttemptView `json:"attempt,omitempty"`
|
||||
// Started says the agent has looked at the module once since it started (its baseline): what runs was taken from its
|
||||
// have, or it was applied.
|
||||
Started bool `json:"started"`
|
||||
// Have is what the module's have key says now; nil when it says nothing.
|
||||
Have *HaveView `json:"have,omitempty"`
|
||||
// Composition is the revision read: the latest of the want's build, or the one whose digest was asked.
|
||||
@@ -32,14 +40,56 @@ type CompositionView struct {
|
||||
Judged JudgedView `json:"judged"`
|
||||
// Diff is every resource that differs from what runs, when asked.
|
||||
Diff []judge.ResourceChange `json:"diff,omitempty"`
|
||||
// Own is, of the node's own composition (module node), its own fields beside what runs names (issue 774: ace's
|
||||
// own composition dropped the provisioner client's account, and nothing read could show it).
|
||||
Own *OwnView `json:"own,omitempty"`
|
||||
}
|
||||
|
||||
// WantView is a want as the verb shows it.
|
||||
// OwnView is the node's own fields of a composition (delivery.NodeOwn) beside what runs of them, and what the node's
|
||||
// composer held in its last pass.
|
||||
type OwnView struct {
|
||||
// Provisioner is the account the provisioner's client runs as, as the revision read names it; RunsProvisioner as
|
||||
// what runs names it. Empty: the declaration names none, and every filled write is refused.
|
||||
Provisioner string `json:"provisioner_account"`
|
||||
RunsProvisioner string `json:"runs_provisioner_account,omitempty"`
|
||||
// Adoption is the revision's adoption: the node adopted, and which of its modules are taken; nil for a converged node.
|
||||
Adoption *declaration.Adoption `json:"adoption,omitempty"`
|
||||
// Identities, Groups and Libraries count what the revision names of each.
|
||||
Identities int `json:"identities,omitempty"`
|
||||
Groups int `json:"groups,omitempty"`
|
||||
Libraries int `json:"libraries,omitempty"`
|
||||
// Held is each module the node's composer held in its last pass, with why.
|
||||
Held map[string]string `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// WantView is a want as the verb shows it: as the agent holds it, at the revision of its key it last read.
|
||||
type WantView struct {
|
||||
FP string `json:"fp"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
Yes string `json:"yes,omitempty"`
|
||||
Removed bool `json:"removed,omitempty"`
|
||||
// Self says the node composes the module itself (ADR 0547 §2).
|
||||
Self bool `json:"self,omitempty"`
|
||||
// Backup is the operator's answer to a failed backup the want carries, again or skip (ADR 0539 §16); empty when none.
|
||||
Backup string `json:"backup,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// Revision is the revision of the want's key the agent read last.
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// AttemptView is the agent's record of a module's last apply that did not end running.
|
||||
type AttemptView struct {
|
||||
FP string `json:"fp"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
// WantRevision and CompositionRevision are the revisions of the want's and the composition's keys it was tried at.
|
||||
WantRevision uint64 `json:"want_revision"`
|
||||
CompositionRevision uint64 `json:"composition_revision"`
|
||||
// Held says it is held for a failed backup; AnsweredRevision is the want's revision the hold took, so an answer
|
||||
// (again or skip) in a later revision releases it.
|
||||
Held bool `json:"held,omitempty"`
|
||||
AnsweredRevision uint64 `json:"answered_revision,omitempty"`
|
||||
// Transient says it failed because the bus did not answer for a moment, and is tried again by the clock.
|
||||
Transient bool `json:"transient,omitempty"`
|
||||
}
|
||||
|
||||
// HaveView is a have as the verb shows it.
|
||||
@@ -106,10 +156,16 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo
|
||||
fp := ""
|
||||
if w.Present {
|
||||
fp = w.Want.FP
|
||||
view.Want = &WantView{FP: w.Want.FP, Composed: w.Want.Composed, Yes: w.Want.Yes, Removed: w.Want.Removed}
|
||||
view.Want = &WantView{FP: w.Want.FP, Composed: w.Want.Composed, Yes: w.Want.Yes, Removed: w.Want.Removed,
|
||||
Self: w.Want.Self, Backup: w.Want.Backup, Words: w.Want.Words, Revision: w.Rev}
|
||||
} else if have != nil {
|
||||
fp = have.FP
|
||||
}
|
||||
if att, ok := a.attempt[module]; ok {
|
||||
view.Attempt = &AttemptView{FP: att.FP, Composed: att.Composed, WantRevision: att.WantRev, CompositionRevision: att.CompRev,
|
||||
Held: att.Held, AnsweredRevision: att.AnsweredRev, Transient: att.Transient}
|
||||
}
|
||||
view.Started = a.checked[module]
|
||||
latest, haveLatest := a.compositionFor(module, fp)
|
||||
r, ran := a.running[module]
|
||||
from, known := a.runningComposition(module)
|
||||
@@ -171,6 +227,17 @@ func (a *Agent) Composition(ctx context.Context, module, digest string, diff boo
|
||||
}
|
||||
view.Runs = rv
|
||||
}
|
||||
if module == NodeModule {
|
||||
o := read.C.Own
|
||||
view.Own = &OwnView{Provisioner: o.Provisioner, Adoption: o.Adoption, Identities: len(o.Identities), Groups: len(o.Groups),
|
||||
Libraries: len(o.Libraries)}
|
||||
if running != nil {
|
||||
view.Own.RunsProvisioner = running.Own.Provisioner
|
||||
}
|
||||
if a.env.Held != nil {
|
||||
view.Own.Held = a.env.Held()
|
||||
}
|
||||
}
|
||||
j := judge.JudgeMove(running, read.C)
|
||||
view.Judged = JudgedView{Judgement: string(j.Judgement), Words: j.Words}
|
||||
if a.env.SelfComposed && a.env.Policy != nil && module != NodeModule && (!w.Present || w.Want.Yes != read.Dig) {
|
||||
|
||||
+107
-10
@@ -610,7 +610,7 @@ func applyScoped(
|
||||
err = fmt.Errorf("%w: %s", errRemovalWaits, said)
|
||||
}
|
||||
default:
|
||||
action, detail, err = remove(ctx, sys, orphan, run, made)
|
||||
action, detail, err = remove(ctx, sys, orphan, run, made, journalDir(windows))
|
||||
}
|
||||
if errors.Is(err, errNoRemoval) && store.IsFormer(orphan.ID) {
|
||||
// **A former target of a kind the host cannot remove is left in place and forgotten,
|
||||
@@ -939,6 +939,13 @@ func applyScoped(
|
||||
}
|
||||
}
|
||||
setRunnerWarnings(warnings)
|
||||
// **Every new image of a module is on the machine before any of its containers is set aside** (design 51, issue 1966
|
||||
// ENG-7). A container replaced after the first one's turn pulled its image only then, and for a module whose first
|
||||
// container serves the store its second image comes from, that pull met the stopped one. A pull that fails
|
||||
// replaces nothing of the module: its containers keep running what they ran, the failure names the image, and what
|
||||
// follows in the module is skipped as it is for any step that did not complete.
|
||||
in.pulled = map[string]bool{}
|
||||
pullFirst(ctx, ordered, d, in.pulled, run, &failures, gated, gatedWhy, log)
|
||||
for i, resource := range ordered {
|
||||
if !orphansRemoved && i == guardFirst {
|
||||
// **Only a guard that is up may let the filter go.** Removing the derived filter's
|
||||
@@ -1572,7 +1579,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
||||
case *declaration.User:
|
||||
return applyUser(ctx, sys, res, run, previous, in.groups, in.handed)
|
||||
case *declaration.Archive:
|
||||
return applyArchive(ctx, res, previous, in.away)
|
||||
return applyArchive(ctx, res, previous, in.away, journalDir(in.windows))
|
||||
case *declaration.Process:
|
||||
return applyProcess(ctx, res, run, changed, previous, in.away)
|
||||
case *declaration.Action:
|
||||
@@ -2483,7 +2490,7 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
|
||||
//
|
||||
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
|
||||
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
made map[string]bool) (string, string, error) {
|
||||
made map[string]bool, journals string) (string, string, error) {
|
||||
switch declaration.Type(a.Type) {
|
||||
case declaration.TypeDirectory, declaration.TypeFile, declaration.TypeArchive:
|
||||
// Removal below a home follows no link an account put there either (novox/hq ADR 0266).
|
||||
@@ -2609,7 +2616,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
case declaration.TypeArchive:
|
||||
// Exactly what it unpacked, and the directories the host made for it once they are empty
|
||||
// (novox/hq issue 162).
|
||||
return removeArchive(a)
|
||||
return removeArchive(a, journals)
|
||||
|
||||
default:
|
||||
return "", "", fmt.Errorf("%w: a %q", errNoRemoval, a.Type)
|
||||
@@ -2759,6 +2766,9 @@ type inputs struct {
|
||||
files map[string]bool
|
||||
// handed is every group a record that went this apply handed to a resource still asking for it.
|
||||
handed Handed
|
||||
// pulled is every image this apply found on the machine, or pulled, before any container was replaced (pullFirst):
|
||||
// applyContainer does not ask for it again.
|
||||
pulled map[string]bool
|
||||
}
|
||||
|
||||
// fileDigest is what a file the container reads holds, by digest.
|
||||
@@ -3165,8 +3175,10 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
||||
// gone: an image the registry does not hold then left the service down and every later apply
|
||||
// failing. Pulled (or found present) first, an unobtainable image fails this resource loudly,
|
||||
// naming the image, and the old container keeps running.
|
||||
if err := ensureImage(ctx, cri, r.Image, run); err != nil {
|
||||
return out, fmt.Errorf("not replacing container %s, which keeps running: %w", r.Name, err)
|
||||
if !in.pulled[r.Image] {
|
||||
if err := ensureImage(ctx, cri, r.Image, run); err != nil {
|
||||
return out, fmt.Errorf("not replacing container %s, which keeps running: %w", r.Name, err)
|
||||
}
|
||||
}
|
||||
// **The running container is set aside, never removed, until its replacement runs** (issue 1945): stopped and
|
||||
// renamed, so its name and ports are free, and started again under its own name when the replacement does not
|
||||
@@ -3469,16 +3481,17 @@ func ensureImage(ctx context.Context, cri, image string, run Runner) error {
|
||||
// already hold — built here, or handed over. There is no registry that answers for it, so
|
||||
// pulling would fail somewhere that names a network problem instead of a missing image.
|
||||
if strings.HasPrefix(image, "sha256:") {
|
||||
return fmt.Errorf(
|
||||
return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf(
|
||||
"image %s is not on this machine, and an image named by its own digest cannot be "+
|
||||
"fetched: nothing serves it. Build it here, or load it, before applying this",
|
||||
image)
|
||||
image)}
|
||||
}
|
||||
started := time.Now()
|
||||
if _, err := run(ctx, cri, "pull", image); err != nil {
|
||||
return fmt.Errorf("pulling image %s: %w", image, err)
|
||||
return &PullError{Image: image, Started: started, Err: fmt.Errorf("pulling image %s: %w", image, err)}
|
||||
}
|
||||
if _, err := run(ctx, cri, "image", "inspect", image); err != nil {
|
||||
return fmt.Errorf("image %s is not present after pulling it: %w", image, err)
|
||||
return &PullError{Image: image, Started: time.Now(), Err: fmt.Errorf("image %s is not present after pulling it: %w", image, err)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -4340,3 +4353,87 @@ func ownerText(info os.FileInfo) string {
|
||||
}
|
||||
return fmt.Sprintf("%d:%d", uid, gid)
|
||||
}
|
||||
|
||||
// pullFirst finds on the machine, or pulls, the image of each container the declaration runs before any is replaced
|
||||
// (ENG-7) for a module of two containers or more. A container that is run once is left to its marker, and a runtime that
|
||||
// does not answer to applyContainer, which says so. The first image that cannot be had fails its container with the image named and gates its module, so none of
|
||||
// its containers is touched; the images found before it stay pulled.
|
||||
func pullFirst(ctx context.Context, ordered []declaration.Resource, d *declaration.Declaration, pulled map[string]bool,
|
||||
run Runner, failures *[]*Error, gated map[string]bool, gatedWhy map[string]string, log func(string)) {
|
||||
// A module of one container already pulls its image before it sets that container aside (applyContainer), and nothing
|
||||
// of it comes before: only a module of two or more has an image pulled after one of its containers was replaced.
|
||||
count := map[string]int{}
|
||||
for _, resource := range ordered {
|
||||
if c, isContainer := resource.(*declaration.Container); isContainer && c.Image != "" && !c.RunOnce && c.Schedule == "" {
|
||||
module, _ := d.ModuleOfID(c.Identity())
|
||||
count[module]++
|
||||
}
|
||||
}
|
||||
var cri string
|
||||
for _, resource := range ordered {
|
||||
c, isContainer := resource.(*declaration.Container)
|
||||
if !isContainer || c.Image == "" || c.RunOnce || c.Schedule != "" || pulled[c.Image] {
|
||||
continue
|
||||
}
|
||||
module, ours := d.ModuleOfID(c.Identity())
|
||||
if count[module] < 2 {
|
||||
continue
|
||||
}
|
||||
if ours && gated[module] {
|
||||
continue
|
||||
}
|
||||
if cri == "" {
|
||||
var err error
|
||||
if cri, err = containerRuntime(ctx, run); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := ensureImage(ctx, cri, c.Image, run); err != nil {
|
||||
err = fmt.Errorf("not replacing container %s, or any other container of its module, which keep running: %w", c.Name, err)
|
||||
*failures = append(*failures, &Error{Resource: c.Identity(), Err: err})
|
||||
log(fmt.Sprintf(" failed %s (%s): %v", c.Identity(), c.Target(), err))
|
||||
if ours {
|
||||
gated[module] = true
|
||||
gatedWhy[module] = "its image " + c.Image + " could not be had, so none of its containers was replaced"
|
||||
}
|
||||
continue
|
||||
}
|
||||
pulled[c.Image] = true
|
||||
}
|
||||
}
|
||||
|
||||
// PullError is an image that could not be found or pulled, as ensureImage fails: what the node agent reads to tell a failed
|
||||
// pull from another failed apply (ENG-8).
|
||||
type PullError struct {
|
||||
Image string
|
||||
// Started is when the pull began, which the node agent compares with when a store's holder settled (ENG-11).
|
||||
Started time.Time
|
||||
Err error
|
||||
}
|
||||
|
||||
// PullErrors are the pull errors of every failure an apply reports, the first among them and the rest in Failures, in the
|
||||
// order they happened: what the node agent tells a failed pull from another failed apply by (ENG-8). Nil for an error
|
||||
// that is no apply's, or whose failures met none.
|
||||
func PullErrors(err error) []*PullError {
|
||||
var out []*PullError
|
||||
var e *Error
|
||||
if errors.As(err, &e) {
|
||||
for _, f := range e.Failures {
|
||||
var p *PullError
|
||||
if errors.As(f.Err, &p) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
var p *PullError
|
||||
if errors.As(err, &p) {
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e *PullError) Error() string { return e.Err.Error() }
|
||||
func (e *PullError) Unwrap() error { return e.Err }
|
||||
|
||||
@@ -35,9 +35,14 @@ import (
|
||||
// enough for a desktop theme and small enough to notice.
|
||||
const maxArchive = 512 << 20
|
||||
|
||||
func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway) (Outcome, error) {
|
||||
func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Applied, away *storeAway, journals string) (Outcome, error) {
|
||||
out := begin(r)
|
||||
out.Action = "unchanged"
|
||||
recorded := previous.Wrote != ""
|
||||
previous, err := withJournal(journals, previous, r.Path)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
body, err := fetchMinding(ctx, r.Source, away)
|
||||
if err != nil {
|
||||
@@ -78,8 +83,12 @@ func applyArchive(ctx context.Context, r *declaration.Archive, previous store.Ap
|
||||
return out, err
|
||||
}
|
||||
out.unpacked = &unpacked
|
||||
// At once, before the rest of the apply can fail or the engine stop (novox/hq issue 1585).
|
||||
if err := keepJournal(journals, r.Path, got, &unpacked); err != nil {
|
||||
return out, fmt.Errorf("%s was unpacked, and what it unpacked could not be written down: %w", r.Path, err)
|
||||
}
|
||||
out.Action = "updated"
|
||||
if previous.Wrote == "" {
|
||||
if !recorded {
|
||||
out.Action = "created"
|
||||
}
|
||||
out.Detail = fmt.Sprintf("%d file(s)", written)
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// What an archive unpacked is written down the moment it is unpacked (novox/hq issue 1585).
|
||||
//
|
||||
// The node's state is saved at the end of a module's apply, so an engine that stopped, was replaced or lost its
|
||||
// state between the unpack and that save held files on the machine it had no record of. The next apply then found
|
||||
// its own files and refused them as ones "the mesh did not put there" (issue 162), and a module could not move to
|
||||
// its new build until a person removed them. So, at once and atomically, the host keeps the digest it unpacked and
|
||||
// the files and directories it placed; an apply that holds no record of the archive reads this instead of treating
|
||||
// everything there as somebody's. The record in the state stays the authority: the journal is read only when there
|
||||
// is none.
|
||||
//
|
||||
// **In the node-engine's own state directory, readable and writable by root alone, never beside the archive.** The
|
||||
// archive's directory may be writable by the module's account (a home, a data directory): a journal there could be
|
||||
// forged, and a root engine would then remove, or take as its own, any path the forger named. And an entry is read
|
||||
// only when it names a path inside the archive's directory (filepath.IsLocal), and a parent only when it is above it.
|
||||
|
||||
// JournalDirName is the directory under the node's state directory that keeps the journals.
|
||||
const JournalDirName = "unpacked"
|
||||
|
||||
type unpackJournal struct {
|
||||
Target string `json:"target"`
|
||||
Wrote string `json:"wrote"`
|
||||
Unpacked *store.Unpacked `json:"unpacked"`
|
||||
}
|
||||
|
||||
// journalDir is where the journals of this apply are kept: under the state directory every applier on the machine
|
||||
// shares (the one its windows are kept in). Empty for a caller with no state directory (a test), which keeps none.
|
||||
func journalDir(w *Windows) string {
|
||||
if w == nil || w.dir == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(filepath.Dir(w.dir), JournalDirName)
|
||||
}
|
||||
|
||||
// journalPath names the journal of the archive at target: by the digest of its path, so no path names a file
|
||||
// outside the journals' directory.
|
||||
func journalPath(dir, target string) string {
|
||||
sum := sha256.Sum256([]byte(filepath.Clean(target)))
|
||||
return filepath.Join(dir, hex.EncodeToString(sum[:])+".json")
|
||||
}
|
||||
|
||||
// keepJournal writes down what an archive unpacked at path.
|
||||
func keepJournal(dir, path, wrote string, u *store.Unpacked) error {
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := json.Marshal(unpackJournal{Target: filepath.Clean(path), Wrote: wrote, Unpacked: u})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAtomicallyByPath(journalPath(dir, path), raw, 0o600)
|
||||
}
|
||||
|
||||
// journalOf is what the journal says of path: false when there is none, and an error when there is one that cannot
|
||||
// be read or names a path outside the archive's directory, which is never believed.
|
||||
func journalOf(dir, path string) (unpackJournal, bool, error) {
|
||||
if dir == "" {
|
||||
return unpackJournal{}, false, nil
|
||||
}
|
||||
raw, err := os.ReadFile(journalPath(dir, path))
|
||||
if err != nil {
|
||||
return unpackJournal{}, false, nil
|
||||
}
|
||||
var j unpackJournal
|
||||
if err := json.Unmarshal(raw, &j); err != nil {
|
||||
return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s is unreadable: %w", path, err)
|
||||
}
|
||||
if j.Wrote == "" || j.Unpacked == nil {
|
||||
return unpackJournal{}, false, fmt.Errorf("the journal of what was unpacked at %s names no digest or files", path)
|
||||
}
|
||||
if err := checkJournal(j, filepath.Clean(path)); err != nil {
|
||||
return unpackJournal{}, false, err
|
||||
}
|
||||
return j, true, nil
|
||||
}
|
||||
|
||||
// checkJournal refuses a journal of another target, an entry that is not a path inside the archive's directory, and
|
||||
// a parent that is not above it.
|
||||
func checkJournal(j unpackJournal, target string) error {
|
||||
if j.Target != target {
|
||||
return fmt.Errorf("the journal kept for %s names %q", target, j.Target)
|
||||
}
|
||||
for _, rel := range append(append([]string(nil), j.Unpacked.Files...), j.Unpacked.Dirs...) {
|
||||
if !filepath.IsLocal(filepath.FromSlash(rel)) {
|
||||
return fmt.Errorf("the journal of what was unpacked at %s names %q, which is not inside it: not believed", target, rel)
|
||||
}
|
||||
}
|
||||
for _, p := range j.Unpacked.Parents {
|
||||
if p = filepath.Clean(p); !filepath.IsAbs(p) || p == target || !strings.HasPrefix(target, p+string(filepath.Separator)) {
|
||||
return fmt.Errorf("the journal of what was unpacked at %s names the parent %q, which is not above it: not believed", target, p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withJournal is previous, or, when the state holds no record of the archive, what the journal says the host
|
||||
// unpacked at path. A journal not believed is an error: the apply stops and says so rather than guess.
|
||||
func withJournal(dir string, previous store.Applied, path string) (store.Applied, error) {
|
||||
if previous.Wrote != "" {
|
||||
return previous, nil
|
||||
}
|
||||
j, ok, err := journalOf(dir, path)
|
||||
if err != nil || !ok {
|
||||
return previous, err
|
||||
}
|
||||
previous.Wrote, previous.Target, previous.Unpacked = j.Wrote, path, j.Unpacked
|
||||
return previous, nil
|
||||
}
|
||||
|
||||
// dropJournal removes the journal of the archive at target, once the archive is taken off; whether the state's
|
||||
// record of it still said what it unpacked or not.
|
||||
func dropJournal(dir, target string) {
|
||||
if dir == "" || target == "" {
|
||||
return
|
||||
}
|
||||
_ = os.Remove(journalPath(dir, target))
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -152,6 +153,98 @@ func TestAnArchiveDoesNotWriteOverAFileItFound(t *testing.T) {
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 1 {
|
||||
t.Fatalf("%d entries beside the directory", len(entries))
|
||||
}
|
||||
|
||||
// Issue 1585: what the host unpacked is written down at once, in its own state directory, so an archive whose
|
||||
// record in the state was lost is still the host's: the next version replaces its files, drops the ones it no
|
||||
// longer has and leaves a file somebody added. A file the host never unpacked still refuses it, and so does a
|
||||
// journal naming a path outside the archive's directory. The journal goes with the archive, record lost or not.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
journaled bool // the first version was unpacked by the host
|
||||
foreign string
|
||||
forged func(j *unpackJournal)
|
||||
refused string
|
||||
}{
|
||||
{"unpacked by the host, record lost", true, "", nil, ""},
|
||||
{"unpacked by the host, somebody's file beside", true, "mine.txt", nil, ""},
|
||||
{"never unpacked by the host", false, "", nil, "did not put there"},
|
||||
{"a journal naming a path outside", true, "", func(j *unpackJournal) {
|
||||
j.Unpacked.Files = append(j.Unpacked.Files, "../../etc/passwd")
|
||||
}, "not inside it"},
|
||||
{"a journal naming a parent not above", true, "", func(j *unpackJournal) {
|
||||
j.Unpacked.Parents = []string{"/etc"}
|
||||
}, "not above it"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
dir, state := t.TempDir(), t.TempDir()
|
||||
windows, journals := WindowsIn(state), filepath.Join(state, JournalDirName)
|
||||
applyIt := func(decl string, known store.State) (store.State, error) {
|
||||
_, after, err := ApplyMindingWindows(context.Background(), archHost(t), declare(t, decl), known,
|
||||
store.OriginDeclared, noServices, nil, nil, nil, windows)
|
||||
return after, err
|
||||
}
|
||||
target := filepath.Join(dir, "tools")
|
||||
first, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "one", "old": "o"}, "")
|
||||
if row.journaled {
|
||||
if _, err := applyIt(first, store.State{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 1 {
|
||||
t.Fatalf("%d entries beside the archive: the journal is the state directory's", len(entries))
|
||||
}
|
||||
info, err := os.Stat(journalPath(journals, target))
|
||||
if err != nil || info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("no journal of root's alone after the unpack: %v %v", info, err)
|
||||
}
|
||||
} else {
|
||||
os.MkdirAll(target, 0o755)
|
||||
os.WriteFile(filepath.Join(target, "bin"), []byte("one"), 0o644)
|
||||
}
|
||||
if row.forged != nil {
|
||||
j, _, err := journalOf(journals, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
row.forged(&j)
|
||||
raw, _ := json.Marshal(j)
|
||||
os.WriteFile(journalPath(journals, target), raw, 0o600)
|
||||
}
|
||||
if row.foreign != "" {
|
||||
os.WriteFile(filepath.Join(target, row.foreign), []byte("x"), 0o644)
|
||||
}
|
||||
second, _ := archiveDecl(t, "m.tools", target, map[string]string{"bin": "two"}, "")
|
||||
// The state is empty: the record of the first unpack was lost.
|
||||
after, err := applyIt(second, store.State{})
|
||||
if row.refused != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
||||
t.Fatalf("refused? %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(target, "bin")); string(got) != "two" {
|
||||
t.Fatalf("bin is %q", got)
|
||||
}
|
||||
if exists(filepath.Join(target, "old")) {
|
||||
t.Fatal("a file the new version no longer has is still there")
|
||||
}
|
||||
if row.foreign != "" && !exists(filepath.Join(target, row.foreign)) {
|
||||
t.Fatal("somebody's file went")
|
||||
}
|
||||
// Taken off, with a record that no longer says what it unpacked: the journal goes too.
|
||||
for i := range after.Resources {
|
||||
after.Resources[i].Unpacked = nil
|
||||
}
|
||||
if _, err := applyIt(`{"id":"other","type":"file","path":"`+dir+`/other","content":"o"}`, after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists(journalPath(journals, target)) {
|
||||
t.Fatal("the journal outlived the archive")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestADirectoryTheHostMadeGoesOnlyWhenEmpty(t *testing.T) {
|
||||
|
||||
@@ -364,7 +364,16 @@ func mergeInto(fresh, path, owner string, files, dirs []string, o ours) (store.U
|
||||
// module's tools from its bundle — sees the whole tree or none of it, never half, and a file it has
|
||||
// open stays readable until it closes it. The runtime is told the module went by its own membership,
|
||||
// not by the files disappearing.
|
||||
func removeArchive(a store.Applied) (string, string, error) {
|
||||
func removeArchive(a store.Applied, journals string) (string, string, error) {
|
||||
action, detail, err := removeArchiveFiles(a)
|
||||
// And its journal (issue 1585), once the record goes: whether the record said what it unpacked or not.
|
||||
if err == nil {
|
||||
dropJournal(journals, a.Target)
|
||||
}
|
||||
return action, detail, err
|
||||
}
|
||||
|
||||
func removeArchiveFiles(a store.Applied) (string, string, error) {
|
||||
if store.IsFormer(a.ID) {
|
||||
// The version before is what a rollback starts (ADR 0141) and what a reader may still have
|
||||
// open; the launcher retires the host's own versions, not the apply (issue 194).
|
||||
|
||||
@@ -159,7 +159,7 @@ func TestRemovalThroughALinkUnderAHomeIsRefused(t *testing.T) {
|
||||
home, outside := aLinkedHome(t)
|
||||
link(t, outside, filepath.Join(home, ".claude"))
|
||||
_, _, err := remove(context.Background(), archHost(t), store.Applied{ID: "m.f", Type: "file",
|
||||
Target: filepath.Join(home, ".claude", "shadow")}, nil, nil)
|
||||
Target: filepath.Join(home, ".claude", "shadow")}, nil, nil, "")
|
||||
mustBeLinkRefusal(t, err)
|
||||
if _, err := os.Stat(filepath.Join(outside, "shadow")); err != nil {
|
||||
t.Fatal("removed through the link")
|
||||
|
||||
@@ -47,6 +47,63 @@ func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testi
|
||||
if !strings.Contains(err.Error(), pinned) {
|
||||
t.Errorf("the failure does not name the image %s: %v", pinned, err)
|
||||
}
|
||||
var pull *PullError
|
||||
if !errors.As(err, &pull) || pull.Image != pinned {
|
||||
t.Errorf("the failure is not a pull error naming %s: %v", pinned, err)
|
||||
}
|
||||
|
||||
// design 51, ENG-7: a module of two containers whose second new image cannot be pulled replaces neither, and says the
|
||||
// image; with both pullable, both are pulled before the first is set aside.
|
||||
second := strings.Replace(pinned, "postgres", "other", 1)
|
||||
two := `{"declaration":1,"resources":[
|
||||
{"id":"a.first","type":"container","name":"first","image":"` + pinned + `","env":{"X":"1"}},
|
||||
{"id":"a.second","type":"container","name":"second","image":"` + second + `","env":{"X":"2"}}
|
||||
]}`
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
unpullable string
|
||||
wantTouched bool
|
||||
wantPulled string
|
||||
}{
|
||||
{"the second image cannot be pulled", second, false, "pull " + pinned},
|
||||
{"both can", "", true, "pull " + pinned + "; pull " + second},
|
||||
} {
|
||||
d := parseTrusted(t, two)
|
||||
var steps []string
|
||||
present := map[string]bool{}
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
args = oldShape(name, args)
|
||||
switch args[0] {
|
||||
case "info":
|
||||
return "27.0\n", nil
|
||||
case "container":
|
||||
return "true\tsome-older-spec", nil
|
||||
case "image":
|
||||
if present[args[len(args)-1]] {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.New("no such image")
|
||||
case "pull":
|
||||
if args[1] == c.unpullable {
|
||||
return "", errors.New("manifest unknown")
|
||||
}
|
||||
present[args[1]] = true
|
||||
steps = append(steps, "pull "+args[1])
|
||||
case "rm", "rename", "stop", "run":
|
||||
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||
got := strings.Join(steps, "; ")
|
||||
touched := strings.Contains(got, "rename") || strings.Contains(got, "run ")
|
||||
if touched != c.wantTouched || !strings.HasPrefix(got, c.wantPulled) {
|
||||
t.Errorf("%s: steps %q (touched %v), want pulls %q first and touched=%v", c.name, got, touched, c.wantPulled, c.wantTouched)
|
||||
}
|
||||
if !c.wantTouched && (err == nil || !strings.Contains(err.Error(), second)) {
|
||||
t.Errorf("%s: the failure does not name %s: %v", c.name, second, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
|
||||
@@ -54,6 +111,16 @@ func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing
|
||||
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
||||
]}`)
|
||||
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
||||
// The seat and holder a routed image names (ENG-10) are no part of what a container is: another holder is the same spec,
|
||||
// so a declaration composed again for it replaces no container.
|
||||
for _, holder := range []string{"distribution", "elsewhere"} {
|
||||
named := parseTrusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"store","type":"container","name":"store","image":"`+pinned+`","image-seat":"mesh-artifact-store","image-holder":"`+holder+`","env":{"PGDATA":"/data"}}
|
||||
]}`)
|
||||
if got := containerSpec(named.Resources[0].(*declaration.Container), inputs{}); got != want {
|
||||
t.Errorf("a container naming the holder %s has another spec than one naming none", holder)
|
||||
}
|
||||
}
|
||||
var order []string
|
||||
created := false
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
|
||||
@@ -815,7 +815,6 @@ func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.Syste
|
||||
if err != nil {
|
||||
return Published{}, failed(StepRegistry, err)
|
||||
}
|
||||
|
||||
// ---- 8. publish -----------------------------------------------------------------------
|
||||
say("publish — the control plane's image gets its first manifest digest")
|
||||
// **The image this mesh built, not the one the installer carried.** The carried one is the
|
||||
|
||||
@@ -69,8 +69,9 @@ func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *decl
|
||||
return nil
|
||||
}
|
||||
|
||||
// meshBuiltImage is an image the mesh built, as it records it: `artifact-store://<module>/<artifact>@sha256:…`.
|
||||
var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
|
||||
// meshBuiltImage is an image the mesh built, as it records it in either form it reads (ADR 0605 §9, issue 1966 ENG-4):
|
||||
// `artifact-store://<module>/<artifact>@sha256:…`, or the registry seat's `container-image-registry/<module>/<artifact>@sha256:…`.
|
||||
var meshBuiltImage = regexp.MustCompile(`(?:artifact-store://|container-image-registry/)([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
|
||||
|
||||
// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this
|
||||
// mesh's registry: the image that carries the snapshot program.
|
||||
|
||||
@@ -139,4 +139,12 @@ func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) {
|
||||
if got, want := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image, repository+"@"+digest; got != want {
|
||||
t.Errorf("the bundle raises the bus on %s; the nats module's server is %s (%s)", got, want, from)
|
||||
}
|
||||
|
||||
// The image the mesh built is read in either form it is recorded in (ENG-4).
|
||||
sum := strings.Repeat("c", 64)
|
||||
for _, said := range []string{"artifact-store://nats/server@sha256:" + sum, "container-image-registry/nats/server@sha256:" + sum} {
|
||||
if m := meshBuiltImage.FindStringSubmatch("build 1: " + said); m == nil || m[1] != "nats/server@sha256:"+sum {
|
||||
t.Errorf("%s is read as %v", said, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,17 +69,24 @@ func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string,
|
||||
func publishAs(ctx context.Context, o Options, d Deps, imageID, repository string,
|
||||
say func(string)) (Published, error) {
|
||||
|
||||
remote := o.Registry + "/" + repository
|
||||
// Pushed to the store's own loopback with no credential (StoreLoopback); recorded under the seat's reach, the
|
||||
// address every node pulls by, with the digest the store gave.
|
||||
loopback, err := StoreLoopback(o.Catalogue)
|
||||
if err != nil {
|
||||
return Published{}, err
|
||||
}
|
||||
remote := loopback + "/" + repository
|
||||
out := Published{Tagged: remote + ":" + genesisTag}
|
||||
reach := func(digest string) string { return o.Registry + "/" + repository + strings.TrimPrefix(digest, remote) }
|
||||
|
||||
// Asked first. A digest already served is a fact about the registry, and re-pushing an image
|
||||
// the registry already holds is asking it to store what it already has under the name it
|
||||
// already has.
|
||||
if held, err := digestOf(ctx, o, d, remote); err != nil {
|
||||
if held, err := digestOf(ctx, o, d, loopback, repository); err != nil {
|
||||
return out, err
|
||||
} else if held != "" {
|
||||
out.Reference, out.Already = held, true
|
||||
say(" already published " + held)
|
||||
out.Reference, out.Already = reach(held), true
|
||||
say(" already published " + out.Reference)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -93,12 +100,12 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin
|
||||
"over the mesh's own network, which is already the encrypted and authenticated "+
|
||||
"thing. A runtime refusing it for being insecure is refusing a registry on %s, "+
|
||||
"which it does not do for a loopback address",
|
||||
out.Tagged, err, o.Registry)
|
||||
out.Tagged, err, loopback)
|
||||
}
|
||||
|
||||
// Read back, from the registry's own answer rather than computed here. What matters is what
|
||||
// the registry will serve for that reference, and only it can say (novox/hq ADR 0018).
|
||||
pinned, err := digestOf(ctx, o, d, remote)
|
||||
pinned, err := digestOf(ctx, o, d, loopback, repository)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -107,10 +114,10 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin
|
||||
"%s was pushed and the registry does not serve it.\n"+
|
||||
"The next step names this module by the digest this was supposed to produce, so "+
|
||||
"there is nothing to name. Check `docker push` and "+
|
||||
"http://%s/v2/%s/tags/list", out.Tagged, o.Registry, repository)
|
||||
"http://%s/v2/%s/tags/list", out.Tagged, loopback, repository)
|
||||
}
|
||||
out.Reference = pinned
|
||||
say(" published " + pinned)
|
||||
out.Reference = reach(pinned)
|
||||
say(" published " + out.Reference)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -121,13 +128,13 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin
|
||||
// whether the registry still has it — a registry whose volume was recreated would leave the
|
||||
// runtime remembering a digest nothing serves, and the module registered against it would pin the
|
||||
// mesh to an image that cannot be pulled.
|
||||
func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) {
|
||||
func digestOf(ctx context.Context, o Options, d Deps, store, repository string) (string, error) {
|
||||
remote := store + "/" + repository
|
||||
asking, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||
status, body, err := d.Fetch(asking,
|
||||
"http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list")
|
||||
status, body, err := d.Fetch(asking, "http://"+store+"/v2/"+repository+"/tags/list")
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err)
|
||||
return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", store, err)
|
||||
}
|
||||
if status == http.StatusNotFound {
|
||||
// Nothing has ever been pushed under this name. An answer, not a failure.
|
||||
@@ -135,14 +142,14 @@ func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, er
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
return "", fmt.Errorf("the registry answered %d when asked what it holds for %s",
|
||||
status, ControlPlaneRepository)
|
||||
status, repository)
|
||||
}
|
||||
var listed struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &listed); err != nil {
|
||||
return "", fmt.Errorf("the registry's answer about %s is not readable: %w",
|
||||
ControlPlaneRepository, err)
|
||||
repository, err)
|
||||
}
|
||||
if !contains(listed.Tags, genesisTag) {
|
||||
return "", nil
|
||||
|
||||
@@ -22,7 +22,8 @@ func publishing(t *testing.T, fetch func(string) (int, string, error),
|
||||
// Each value named before it is returned: a return of several composite literals spanning lines is laid
|
||||
// out differently by the gofmt of one Go release and the next, and the build seat's is the one that
|
||||
// judges (novox/hq issue 286).
|
||||
options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0}
|
||||
options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0,
|
||||
Catalogue: catalogueWith(t, RegistryModule, upstreamRegistryManifest)}
|
||||
deps := Deps{
|
||||
Run: runtime.run,
|
||||
Fetch: func(_ context.Context, url string) (int, string, error) {
|
||||
|
||||
@@ -95,9 +95,14 @@ func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlan
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// waitForTheRegistry asks `/v2/` until it answers.
|
||||
// waitForTheRegistry asks the store's own `/v2/`, on its loopback (StoreLoopback), until it answers: the pushes that
|
||||
// follow go there, and the store's front is not served yet on a fresh mesh.
|
||||
func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) {
|
||||
where := "http://" + o.Registry + "/v2/"
|
||||
loopback, err := StoreLoopback(o.Catalogue)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
where := "http://" + loopback + "/v2/"
|
||||
|
||||
deadline := time.Now().Add(o.Wait)
|
||||
var last string
|
||||
@@ -207,3 +212,43 @@ func containerIn(manifest []byte, id string) (name, image string, err error) {
|
||||
"the %s module declares no container %q, so the installer does not know what to wait for. "+
|
||||
"It declares: %s", m.Module, id, strings.Join(containers, ", "))
|
||||
}
|
||||
|
||||
// storeContainerPort is the port the registry image listens on inside its container (Distribution's own default).
|
||||
const storeContainerPort = "5000"
|
||||
|
||||
// StoreLoopback is host:port where this machine reaches the store's container itself, read from the port its manifest
|
||||
// publishes for it: on a fresh mesh the bootstrap pushes the holders' own images there, with no credential, because the
|
||||
// store's front (distribution's program, a bundle the mesh builds) is not served until the builder has built it (the
|
||||
// publish contract allows a push on the store's own node to its loopback). A port published on every address is reached
|
||||
// on 127.0.0.1. What the bootstrap records is named under the seat's reach (Options.Registry), never this address.
|
||||
func StoreLoopback(catalogue string) (string, error) {
|
||||
manifest, err := readManifest(catalogue, RegistryModule)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
Ports []string `json:"ports"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return "", fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID != registryResource {
|
||||
continue
|
||||
}
|
||||
for _, p := range r.Ports {
|
||||
parts := strings.Split(strings.SplitN(p, "/", 2)[0], ":")
|
||||
switch {
|
||||
case len(parts) == 3 && parts[2] == storeContainerPort:
|
||||
return parts[0] + ":" + parts[1], nil
|
||||
case len(parts) == 2 && parts[1] == storeContainerPort:
|
||||
return "127.0.0.1:" + parts[0], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("the %s module's container %q publishes no port for the store's %s, so the bootstrap cannot "+
|
||||
"reach the store to push to it", RegistryModule, registryResource, storeContainerPort)
|
||||
}
|
||||
|
||||
@@ -226,8 +226,15 @@ func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) {
|
||||
// moving the trust after the push, or dropping it, fails this test.
|
||||
func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`)
|
||||
o.Catalogue = catalogueWith(t, RegistryModule, upstreamRegistryManifest)
|
||||
// The store as distribution's manifest declares it: its container on this machine's loopback, its front not yet served,
|
||||
// and its state directory, where no credential is ever written (design 51 *On a fresh mesh*, a design limit of issue 1966).
|
||||
stateDir := t.TempDir()
|
||||
o.Catalogue = catalogueWith(t, RegistryModule, strings.NewReplacer(
|
||||
`"path": "/var/lib/mesh/registry"`, `"path": "`+stateDir+`"`,
|
||||
`"ports": ["5000:5000"]`, `"ports": ["127.0.0.1:5001:5000"]`).Replace(upstreamRegistryManifest))
|
||||
o.Registry = "192.0.2.10:5100"
|
||||
const loopback = "127.0.0.1:5001"
|
||||
var fetched []string
|
||||
|
||||
pushed := false
|
||||
mesh := aMeshThatAgrees(nil)
|
||||
@@ -241,7 +248,8 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
case "tag":
|
||||
return "", nil
|
||||
case "push":
|
||||
trusted := false
|
||||
// The runtime trusts a loopback registry with no configuration; any other only once told.
|
||||
trusted := strings.HasPrefix(args[1], "127.0.0.1:")
|
||||
for _, n := range runtime.trusted {
|
||||
trusted = trusted || strings.HasPrefix(args[1], n+"/")
|
||||
}
|
||||
@@ -251,12 +259,13 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
pushed = true
|
||||
return "", nil
|
||||
case "image":
|
||||
return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
||||
return `["` + loopback + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
||||
}
|
||||
}
|
||||
return mesh(name, args)
|
||||
}
|
||||
d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) {
|
||||
fetched = append(fetched, url)
|
||||
if strings.HasSuffix(url, "/v2/") || pushed {
|
||||
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
|
||||
}
|
||||
@@ -287,6 +296,23 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) {
|
||||
t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v",
|
||||
reloaded, confirmed, firstPush, runtime.commands)
|
||||
}
|
||||
// The push went to the store on its loopback, and nothing the run asked or pushed reached the seat's reach; the
|
||||
// reference is named under the reach all the same, and no credential file was written for the store's holder.
|
||||
for _, c := range runtime.commands {
|
||||
if strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push") {
|
||||
if !strings.Contains(c, " "+loopback+"/mesh-controller:") {
|
||||
t.Errorf("a push not to the loopback store: %q", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, u := range fetched {
|
||||
if !strings.HasPrefix(u, "http://"+loopback+"/v2/") {
|
||||
t.Errorf("the run asked %s, not the loopback store", u)
|
||||
}
|
||||
}
|
||||
if entries, err := os.ReadDir(stateDir); err != nil || len(entries) != 0 {
|
||||
t.Errorf("the run wrote into the store's state directory: %v (%v)", entries, err)
|
||||
}
|
||||
if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" {
|
||||
t.Errorf("the result says it trusted %v", result.RegistryTrusted)
|
||||
}
|
||||
|
||||
@@ -63,6 +63,15 @@ func CompositionsIn(in Input, node string, fps map[string]string, mode Mode) (ou
|
||||
return compositions(in, node, fps, mode, false)
|
||||
}
|
||||
|
||||
// Folds says whether a module the node runs inside its own composition (inOwn, Node.InOwn) is folded there at the
|
||||
// fingerprint fp it is composed at: only at no fingerprint of the new form. At one (src2:) it has a composition of its
|
||||
// own, composed from its build, so every reader of a build (frombus) takes the build in exactly where this says it is
|
||||
// not folded (issue 1585: nats on novox, folded by the reader and composed at src2: here, refused for its server
|
||||
// artifact no build had published).
|
||||
func Folds(inOwn bool, fp string) bool {
|
||||
return inOwn && !strings.HasPrefix(fp, delivery.SourcePrefix2)
|
||||
}
|
||||
|
||||
func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]delivery.Composed, skipped, stood []string, err error) {
|
||||
reading := in
|
||||
reading.Nodes = append([]Node(nil), in.Nodes...)
|
||||
@@ -84,7 +93,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli
|
||||
// The private network's module is built by nobody: no fingerprint of the new form ever names it.
|
||||
inOwn[overlay.Name] = true
|
||||
folds := func(module string) bool {
|
||||
return module != "" && inOwn[module] && !strings.HasPrefix(fps[module], delivery.SourcePrefix2)
|
||||
return module != "" && Folds(inOwn[module], fps[module])
|
||||
}
|
||||
if !found {
|
||||
return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
|
||||
|
||||
@@ -191,6 +191,28 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) {
|
||||
!strings.Contains(held["spiegel"], "spiegel has not been built for this mesh") {
|
||||
t.Errorf("composed spiegel without what runs: %v %v, want spiegel held alone", err, held)
|
||||
}
|
||||
// Issue 1585 (nats on novox, met twice): wanted at a build of the new form, a module in InOwn is no longer folded
|
||||
// (compose.Folds), so its build is taken in and it composes with its image, never from the indexer's manifest as
|
||||
// "not built for this mesh".
|
||||
{
|
||||
bb := mk("lemurs", overlay.Name, "spiegel")
|
||||
bp := bb.placements["n1"]
|
||||
bp.Decided.Endpoint = "n1.example:51820"
|
||||
bb.placements["n1"] = bp
|
||||
sp := bb.modules["spiegel"]
|
||||
sp.FP2 = fp("3")
|
||||
bb.modules["spiegel"] = sp
|
||||
image := "registry.example:5100/spiegel/site@sha256:" + strings.Repeat("7", 64)
|
||||
bb.builds["spiegel "+sp.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: sp.Manifest,
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "upstream", Reference: image, Digest: "sha256:" + strings.Repeat("7", 64)}}}
|
||||
read, err := ComposingInput(ctx, bb, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}},
|
||||
map[string]string{"lemurs": fp("e"), "spiegel": sp.FP2})
|
||||
raw, _ := json.Marshal(read.Catalogue["spiegel"])
|
||||
if err != nil || read.ComposedHow.Held["spiegel"] != "" || !strings.Contains(string(raw), strings.Repeat("7", 64)) {
|
||||
t.Errorf("spiegel in InOwn wanted at %s: %v, held %v, read as %s; want its build's image", sp.FP2, err,
|
||||
read.ComposedHow, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// (C) Issue 1126, C5: what the others read of a module carried as it runs is what runs, never its stub's: the filter
|
||||
// opens the machine port its running container publishes (de-spiegel's 20005 on novox, not its software's 8080; a
|
||||
|
||||
@@ -98,9 +98,10 @@ type SignersReader interface {
|
||||
|
||||
// HandedReader is a Reader that also says, of the composing node's modules, which the vault judged handed to the
|
||||
// provisioner when it answered OwnSecrets (the vault's `own`, its `handed`: catalogue.AssignmentHands of the plan in
|
||||
// each module's assignment). The node's composition then hands exactly those (compose.Node.Handed, issue 774), so the
|
||||
// vault and the node never judge one module two ways. Nil, or a Reader without it (a vault older than the field),
|
||||
// composes by the manifest and the machine, as before.
|
||||
// each module's assignment). Never the rule: the node hands by the manifest, its build and the machine
|
||||
// (resolution.Handing, issue 1966), and the plans it composes are what the vault reads next; the vault's word is said
|
||||
// where the two disagree (resolution.VaultDisagrees). Nil, or a Reader without it (a vault older than the field), says
|
||||
// nothing.
|
||||
type HandedReader interface {
|
||||
Handed(ctx context.Context, node string) map[string]bool
|
||||
}
|
||||
@@ -234,6 +235,7 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module
|
||||
}
|
||||
in, err := b.finish(ctx, r, r.Build, mesh, s, b.in, refuse, wanted)
|
||||
if err == nil {
|
||||
runningProvisioner(&in, s.Node, s.Running)
|
||||
// What runs is kept while a manifest does not yet say what shared code no longer infers (ADR 0601 §4).
|
||||
for _, k := range keepWhatRuns(&in, s.Node, s.Running) {
|
||||
if s.Kept != nil {
|
||||
@@ -876,7 +878,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m
|
||||
keptAt := maps.Clone(s.KeptAt)
|
||||
for _, pm := range p.Modules {
|
||||
if m, ok := modules[pm.Module]; ok && m.FP2 != "" && wanted[pm.Module] != "" && wanted[pm.Module] != m.FP2 &&
|
||||
keptAt[pm.Module] == "" && !slices.Contains(s.InOwn, pm.Module) {
|
||||
keptAt[pm.Module] == "" && !compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) {
|
||||
if keptAt == nil {
|
||||
keptAt = map[string]string{}
|
||||
}
|
||||
@@ -891,9 +893,10 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m
|
||||
}
|
||||
for _, pm := range p.Modules {
|
||||
m, ok := modules[pm.Module]
|
||||
if !ok || m.FP2 == "" || keptAt[pm.Module] != "" || slices.Contains(s.InOwn, pm.Module) {
|
||||
// A module it runs inside its own composition is composed there as before: no build of the new form is what
|
||||
// it runs (Scope.InOwn).
|
||||
if !ok || m.FP2 == "" || keptAt[pm.Module] != "" || compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) {
|
||||
// A module it runs inside its own composition is composed there as before while it is folded: no build of the
|
||||
// new form is what it runs (Scope.InOwn). Wanted at a build of the new form it is no longer folded (compose.Folds,
|
||||
// the composition's own rule), so its build is taken in (issue 1585).
|
||||
continue
|
||||
}
|
||||
if _, refused := in.Refused[pm.Module]; refused {
|
||||
@@ -1230,3 +1233,38 @@ func withoutHeldData(in compose.Input, node string) compose.Input {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runningProvisioner names, on node, the account its provisioner client runs as now (compose.Node.RunningProvisioner): the
|
||||
// account its own composition as it runs names, else the user of the running process of the module that runs the client.
|
||||
func runningProvisioner(in *compose.Input, node string, running []delivery.Composed) {
|
||||
account := ""
|
||||
for _, c := range running {
|
||||
if c.Module == "" && c.Own.Provisioner != "" {
|
||||
account = c.Own.Provisioner
|
||||
}
|
||||
}
|
||||
for _, c := range running {
|
||||
if account != "" {
|
||||
break
|
||||
}
|
||||
man, ok := in.Catalogue[c.Module]
|
||||
if c.Module == "" || !ok || !man.ClaimsSeat(compose.NodeProvisionerSeat) {
|
||||
continue
|
||||
}
|
||||
for _, raw := range c.Resources {
|
||||
var r struct {
|
||||
Type string `json:"type"`
|
||||
User string `json:"user"`
|
||||
}
|
||||
if json.Unmarshal(raw, &r) == nil && r.Type == "process" && r.User != "" && r.User != "root" {
|
||||
account = r.User
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range in.Nodes {
|
||||
if in.Nodes[i].Name == node {
|
||||
in.Nodes[i].RunningProvisioner = account
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ package frombustest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -204,6 +206,92 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) {
|
||||
if fetched-archives != 1 {
|
||||
t.Errorf("restic's archive was read %d times over two compositions, want once", fetched-archives)
|
||||
}
|
||||
|
||||
// Issue 774 (ace, 2026-11-04 01:29): the provisioner client held as its stub (its build not done, so the node keeps
|
||||
// running what it runs) keeps the account the node's own composition names, and restic stays handed to it with its
|
||||
// plan; without it every filled write was refused and no module was handed.
|
||||
if b, err = Provisioned(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fps, ownKey := FPs(), ""
|
||||
ownKey, _ = delivery.ComposedNodeKey(Node)
|
||||
resticKey, _ := delivery.ComposedKey(Node, "restic", fps["restic"])
|
||||
before, _, _, err := frombus.Compose(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}, fps)
|
||||
account := before[ownKey].Own.Provisioner
|
||||
if err != nil || account == "" || len(before[resticKey].Filled) == 0 {
|
||||
t.Fatalf("the node composes with its client: account %q, restic's filled files %v: %v", account, before[resticKey].Filled, err)
|
||||
}
|
||||
var running []delivery.Composed
|
||||
for _, c := range before {
|
||||
running = append(running, c)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
own, placed bool // the account read from the running own composition, else the client's running process; the client placed
|
||||
userless bool // the client placed and not held, its process naming no user: it composes no account
|
||||
// filledHeld holds every module with a filled file (restic, its build not done) and not the client: the node still
|
||||
// names the account (issue 774, ace 2026-11-04: its build modules and ssh-client held, no filled file, no account)
|
||||
filledHeld bool
|
||||
}{{true, true, false, false}, {false, true, false, false}, {true, false, false, false}, {false, false, false, false},
|
||||
{true, true, true, false}, {true, true, false, true}} {
|
||||
own := row.own
|
||||
runs := running
|
||||
if !own { // what runs names no account of its own: the client's running process says it
|
||||
runs = nil
|
||||
for _, c := range running {
|
||||
if c.Module == "" {
|
||||
c.Own.Provisioner = ""
|
||||
}
|
||||
runs = append(runs, c)
|
||||
}
|
||||
}
|
||||
bb, err := Provisioned()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := account
|
||||
if row.userless {
|
||||
// placed and not held, composing no account of its own: no stale account is kept from what runs
|
||||
mod := bb.Module["mesh-provisioner-client"]
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(bb.Manifests[mod.Manifest], &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m["resources"].([]any) {
|
||||
delete(r.(map[string]any), "user")
|
||||
}
|
||||
raw, _ := json.Marshal(m)
|
||||
mod.Manifest = delivery.ManifestDigest(raw)
|
||||
bb.Module["mesh-provisioner-client"], bb.Manifests[mod.Manifest] = mod, raw
|
||||
built := *bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]]
|
||||
built.Manifest = mod.Manifest
|
||||
bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] = &built
|
||||
want = ""
|
||||
} else if row.filledHeld {
|
||||
bb.Builds["restic "+fps["restic"]] = &delivery.BuilderBuild{State: delivery.BuildBuilding}
|
||||
} else if row.placed {
|
||||
// held: its build not done while it runs
|
||||
bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] = &delivery.BuilderBuild{State: delivery.BuildBuilding}
|
||||
} else {
|
||||
// taken out of the node's placement while what runs still names it: the account goes with it, never kept stale
|
||||
p := bb.Placement[Node]
|
||||
p.Modules = slices.DeleteFunc(slices.Clone(p.Modules), func(m delivery.PlacedModule) bool { return m.Module == "mesh-provisioner-client" })
|
||||
bb.Placement[Node] = p
|
||||
want = ""
|
||||
}
|
||||
after, _, held, err := frombus.Compose(context.Background(), bb, frombus.Mesh{}, frombus.Scope{Node: Node, Running: runs}, fps)
|
||||
if _, isHeld := held["mesh-provisioner-client"]; err != nil || isHeld != (row.placed && !row.userless && !row.filledHeld) {
|
||||
t.Fatalf("%+v: held %v: %v", row, held, err)
|
||||
}
|
||||
if got := after[ownKey].Own.Provisioner; got != want {
|
||||
t.Errorf("%+v: the node's own names %q, want %q", row, got, want)
|
||||
}
|
||||
if _, resticHeld := held["restic"]; row.filledHeld && !resticHeld {
|
||||
t.Errorf("%+v: restic held %v", row, resticHeld)
|
||||
}
|
||||
if row.placed && !row.userless && !row.filledHeld && !reflect.DeepEqual(after[resticKey].Filled, before[resticKey].Filled) {
|
||||
t.Errorf("%+v: restic's filled files %v, want %v", row, after[resticKey].Filled, before[resticKey].Filled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// statesOf is the bus b as a module's read states hold it.
|
||||
|
||||
@@ -159,8 +159,9 @@ type Node struct {
|
||||
// Assigned is the modules assigned to it, in the order the controller reads them (inv.Assigned).
|
||||
Assigned []string
|
||||
// Handed is, by module, whether the vault judged it handed to the provisioner when it answered this node's own
|
||||
// secrets (catalogue.AssignmentHands of the plan in its assignment): the node's own composition hands exactly those
|
||||
// (catalogue.Rendering.Assigned, issue 774). Nil where the vault said nothing of it, as for the controller.
|
||||
// secrets (catalogue.AssignmentHands of the plan in its assignment). Never the rule (resolution.Handing, issue 1966):
|
||||
// read only to say where the vault and this composition disagree (resolution.VaultDisagrees). Nil where the vault said
|
||||
// nothing of it, as for the controller.
|
||||
Handed map[string]bool
|
||||
// Capabilities are the ones present on it (inv.ProfileOf).
|
||||
Capabilities map[string]bool
|
||||
@@ -224,6 +225,11 @@ type Node struct {
|
||||
// so it is left to the node: Compositions composes it as a stub (Stub) and carries its resources as they run, each
|
||||
// after the resource it runs after (issue 1126). Nil composes every folded module from its manifest, as before.
|
||||
RunsOwn *delivery.Composed
|
||||
// RunningProvisioner is the account the provisioner's client runs as on this node now, read from what runs (its own
|
||||
// composition's provisioner account, else the user of the provisioner client's running process), empty when nothing
|
||||
// says it. A client held as its stub this pass composes no process, so the node keeps this account and its handing
|
||||
// (provisionerAccount; issue 774: ace, 2026-11-04 01:29, every filled write refused and postgres's consumers unhanded).
|
||||
RunningProvisioner string
|
||||
// Held is the modules refused alone on this node, each composed as what runs of it (Carried, in Kept; its Stub when
|
||||
// nothing of it runs) so the rest is composed (novox/hq issue 1126): none of a held module's own composition is in what
|
||||
// Compositions answers, so the node keeps what it last applied of it; one whose resources the node's own runs
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package compose
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// What an input says of its held modules on the wire (issue 774, the group check of feat/l3-final 278aa5f0ccbe). A held
|
||||
// module stands in the input as its stub (Stub), and the stub's mark (catalogue.Manifest.Stubbed) is no field of a
|
||||
// manifest's JSON: an input a node-engine sent the check's sandbox lost it, so the sandbox read node-tools, held by its
|
||||
// manifest's verdict, as a tool runner with no bundle and refused every module of ace, g14 and shanks for "its build
|
||||
// produced 0 bundle(s)" instead of naming the verdict. So the input carries the mark itself, by module, for its catalogue
|
||||
// and for each node's kept manifests, and a reader marks each again. An input from a node-engine before the mark (no
|
||||
// Held) is read by its contract: a module Refused names stands in Catalogue as its stub.
|
||||
|
||||
// heldMarks are the modules an input holds as their stubs: those of its catalogue, and those each node keeps, by node.
|
||||
type heldMarks struct {
|
||||
Catalogue []string `json:",omitempty"`
|
||||
Kept map[string][]string `json:",omitempty"`
|
||||
}
|
||||
|
||||
// inputFields is Input without its JSON methods.
|
||||
type inputFields Input
|
||||
|
||||
// inputWire is an input as it crosses the bus: its fields, and the modules it holds as their stubs.
|
||||
type inputWire struct {
|
||||
inputFields
|
||||
Held *heldMarks `json:",omitempty"`
|
||||
}
|
||||
|
||||
// MarshalJSON writes the input with the modules it holds as their stubs named beside it (heldMarks).
|
||||
func (in Input) MarshalJSON() ([]byte, error) {
|
||||
w := inputWire{inputFields: inputFields(in)}
|
||||
marks := heldMarks{Catalogue: stubbedOf(in.Catalogue)}
|
||||
for _, n := range in.Nodes {
|
||||
if kept := stubbedOf(n.Kept); len(kept) > 0 {
|
||||
if marks.Kept == nil {
|
||||
marks.Kept = map[string][]string{}
|
||||
}
|
||||
marks.Kept[n.Name] = kept
|
||||
}
|
||||
}
|
||||
if len(marks.Catalogue) > 0 || len(marks.Kept) > 0 {
|
||||
w.Held = &marks
|
||||
}
|
||||
return json.Marshal(w)
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads an input and marks again each module it holds as its stub: those it names held, and, by the
|
||||
// input's contract, every module Refused names that its catalogue holds.
|
||||
func (in *Input) UnmarshalJSON(raw []byte) error {
|
||||
var w inputWire
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
return err
|
||||
}
|
||||
*in = Input(w.inputFields)
|
||||
held := map[string]bool{}
|
||||
for m := range in.Refused {
|
||||
held[m] = true
|
||||
}
|
||||
if w.Held != nil {
|
||||
for _, m := range w.Held.Catalogue {
|
||||
held[m] = true
|
||||
}
|
||||
}
|
||||
for m := range held {
|
||||
if man, ok := in.Catalogue[m]; ok {
|
||||
man.Stubbed = true
|
||||
in.Catalogue[m] = man
|
||||
}
|
||||
}
|
||||
if w.Held != nil {
|
||||
for i := range in.Nodes {
|
||||
for _, m := range w.Held.Kept[in.Nodes[i].Name] {
|
||||
if man, ok := in.Nodes[i].Kept[m]; ok {
|
||||
man.Stubbed = true
|
||||
in.Nodes[i].Kept[m] = man
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stubbedOf is each module of manifests held as its stub, sorted.
|
||||
func stubbedOf(manifests map[string]catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for name, m := range manifests {
|
||||
if m.Stubbed {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -115,6 +115,11 @@ func (d Declared) Body() ([]byte, error) {
|
||||
}
|
||||
if len(d.Filled) > 0 {
|
||||
envelope["filled_files"] = d.Filled
|
||||
}
|
||||
// The provisioner client's account is the node's, named whenever a client runs here, filled files or not: a node
|
||||
// whose every filled-file module is held this pass names none of its files, and still keeps them, so the client's
|
||||
// writes of them must be accepted (issue 774, ace 2026-11-04: its build modules and ssh-client held, no account).
|
||||
if d.Provisioner != "" {
|
||||
envelope["provisioner_account"] = d.Provisioner
|
||||
}
|
||||
// An empty declaration is deliberate: the node owns nothing the mesh put there (novox/hq issue 127), and the host refuses an
|
||||
@@ -202,9 +207,7 @@ func (c *composer) declaration(nodeName string, mode Mode) (Declared, error) {
|
||||
Filled: composed.Filled, LocalNetwork: resolution.LocalNetworkPorts(rules, with.LocalNetworks),
|
||||
CoreBoxesWanted: rendered.CoreBoxesWanted, Said: rendered.Said,
|
||||
Sequence: node.Sequence, Epoch: node.Epoch, Generation: node.Generation}
|
||||
if len(composed.Filled) > 0 {
|
||||
out.Provisioner = with.Provisioner
|
||||
}
|
||||
out.Provisioner = with.Provisioner
|
||||
// A handed module only pulled in by another keeps what requires it, for its value of its own in the assignments mirror (issue 765).
|
||||
for module := range composed.Plans {
|
||||
if by := resolution.RequiredBy(plan, module); by != "" {
|
||||
|
||||
@@ -74,6 +74,12 @@ func (c *composer) places(nodeName string, resolved Resolved) (Placed, error) {
|
||||
// not sent the field they would refuse.
|
||||
r.Hands = takesHanded(node)
|
||||
r.Provisioner = provisionerAccount(plan)
|
||||
if r.Provisioner == "" && provisionerHeld(plan) {
|
||||
// The provisioner's client is placed here and held this pass (its stub, or carried as it runs, composes no process
|
||||
// of its own): it keeps running as it runs, so the account it runs as stays named, and every module stays handed
|
||||
// to it. A client not held that composes no account, or none placed, names none: no stale account is kept.
|
||||
r.Provisioner = node.RunningProvisioner
|
||||
}
|
||||
r.Assigned = node.Handed
|
||||
r.ReadsHealth = node.Stated && node.Contract >= ReadinessContract
|
||||
r.JudgesRoot = node.Stated && node.Contract >= RootContract
|
||||
@@ -161,7 +167,21 @@ func (c *composer) places(nodeName string, resolved Resolved) (Placed, error) {
|
||||
|
||||
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR 0222).
|
||||
if store != "" {
|
||||
r.SeatReach = map[string]string{"mesh-artifact-store": store}
|
||||
// Both names the store is reached through answer the one address (ENG-3), each with its holder from the holdings on
|
||||
// record, written beside every image routed through it (design 51, ENG-10).
|
||||
r.SeatReach, r.SeatHolder = map[string]string{}, map[string]string{}
|
||||
for _, seat := range []string{"mesh-artifact-store", "container-image-registry"} {
|
||||
r.SeatReach[seat] = store
|
||||
for _, h := range c.in.Holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat && h.Scope == catalogue.ScopeMesh && h.Module != "" {
|
||||
r.SeatHolder[seat] = h.Module
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(r.SeatHolder) == 0 {
|
||||
r.SeatHolder = nil
|
||||
}
|
||||
}
|
||||
r.Settings = settings
|
||||
r.Given = given
|
||||
@@ -190,6 +210,21 @@ func provisionerAccount(plan catalogue.Resolution) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// provisionerHeld says the plan places a module claiming the provisioner client's seat on the machine that this pass
|
||||
// holds (its stand-in: Stub, or Carried as it runs), which composes none of the client's process.
|
||||
func provisionerHeld(plan catalogue.Resolution) bool {
|
||||
for _, m := range plan.Modules {
|
||||
if m.Stubbed && m.ClaimsSeat(NodeProvisionerSeat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RunsTheProvisionerClient says a module runs its machine's provisioner client (runsTheProvisionerClient), for a reader
|
||||
// of what runs.
|
||||
func RunsTheProvisionerClient(m catalogue.Manifest) bool { return runsTheProvisionerClient(m) }
|
||||
|
||||
// runsTheProvisionerClient says a module runs its machine's provisioner client: it claims the seat and declares the own
|
||||
// secret that account is delivered in.
|
||||
func runsTheProvisionerClient(m catalogue.Manifest) bool {
|
||||
|
||||
@@ -272,6 +272,22 @@ func TestTheArtifactStoreIsReachedOverTheNetworkOrOnLoopbackForItsHolder(t *test
|
||||
if got := placed(t, in, "anchor").Rendering.Built; !got["mesh-sdk"] {
|
||||
t.Errorf("built is %v", got)
|
||||
}
|
||||
// Its holder, from the holdings on record, beside the address (design 51, ENG-10): none on record names none, and a change
|
||||
// of the holder is the rendering's.
|
||||
if got := placed(t, in, "anchor").Rendering.SeatHolder; got != nil {
|
||||
t.Errorf("a store with no holder on record names %v", got)
|
||||
}
|
||||
in.Holdings = []catalogue.Held{{Claim: "mesh-artifact-store", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "artifacts"}}
|
||||
if got := placed(t, in, "laptop").Rendering.SeatHolder["mesh-artifact-store"]; got != "" {
|
||||
t.Errorf("a machine that reaches no store names its holder %q", got)
|
||||
}
|
||||
if got := placed(t, in, "anchor").Rendering.SeatHolder["mesh-artifact-store"]; got != "artifacts" {
|
||||
t.Errorf("the store's holder is %q", got)
|
||||
}
|
||||
in.Holdings[0].Module = "distribution"
|
||||
if got := placed(t, in, "anchor").Rendering.SeatHolder["mesh-artifact-store"]; got != "distribution" {
|
||||
t.Errorf("after the holder changed it is %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A zone a module declares is placed where the mesh put it: settled from that node's settings, at the node's private address, on
|
||||
|
||||
@@ -41,6 +41,7 @@ func artifactsInto(resource map[string]any, module string, with mcat.Rendering)
|
||||
module, resource["id"], written, mcat.ArtifactStoreProvision)
|
||||
}
|
||||
resource[key] = mcat.Routed(written, with.ArtifactStore)
|
||||
imageSeatInto(resource, key, written, with)
|
||||
continue
|
||||
}
|
||||
if with.ArtifactStore == "" {
|
||||
@@ -59,7 +60,32 @@ func artifactsInto(resource map[string]any, module string, with mcat.Rendering)
|
||||
}
|
||||
if with.Built[repository] {
|
||||
resource[key] = mcat.Routed(recorded, with.ArtifactStore)
|
||||
imageSeatInto(resource, key, written, with)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// artifactSeat is the seat an image is routed through: the artifact store's (the registries' four seats follow as the
|
||||
// references name them).
|
||||
const artifactSeat = "mesh-artifact-store"
|
||||
|
||||
// imageSeatInto writes beside a container's routed image the seat the reference came through and that seat's holder (design 51
|
||||
// *distribution's first delivery*, issue 1966 ENG-10), so the node-engine finds by the declaration whom a failed pull waits
|
||||
// on, never by a module's name. Neither is part of what a container is: a declaration composed again for a new holder
|
||||
// replaces no container. Nothing is written where the holder is not on record, or for a resource that is no container.
|
||||
func imageSeatInto(resource map[string]any, key, written string, with mcat.Rendering) {
|
||||
if key != "image" || resource["type"] != "container" {
|
||||
return
|
||||
}
|
||||
// The seat a reference of the registry form names; the older forms came through the artifact store's.
|
||||
seat := artifactSeat
|
||||
if named, _, ok := mcat.InRegistrySeat(written); ok {
|
||||
seat = named
|
||||
}
|
||||
if with.SeatHolder[seat] == "" {
|
||||
return
|
||||
}
|
||||
resource["image-seat"] = seat
|
||||
resource["image-holder"] = with.SeatHolder[seat]
|
||||
}
|
||||
|
||||
@@ -46,6 +46,45 @@ func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) {
|
||||
t.Errorf("composing wrote the address into the catalogue's copy: %v", got)
|
||||
}
|
||||
|
||||
// The seat the image came through and its holder, beside it (ENG-10); a holder not on record writes neither, a public
|
||||
// image and an archive carry none, and a holder changing changes the declaration but not the container's spec.
|
||||
if got := fileNamed(out, "gitea.server"); got["image-seat"] != nil || got["image-holder"] != nil {
|
||||
t.Errorf("a holder not on record was written: %v %v", got["image-seat"], got["image-holder"])
|
||||
}
|
||||
held := mcat.Rendering{ArtifactStore: "anchor.internal:5101", SeatHolder: map[string]string{"mesh-artifact-store": "distribution"}}
|
||||
held1, err := Declaration(r, held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(held1, "gitea.server"); got["image-seat"] != "mesh-artifact-store" || got["image-holder"] != "distribution" {
|
||||
t.Errorf("the routed image names its seat and holder as %v %v", got["image-seat"], got["image-holder"])
|
||||
}
|
||||
for _, id := range []string{"gitea.config", "gitea.cache"} {
|
||||
if got := fileNamed(held1, id); got["image-seat"] != nil || got["image-holder"] != nil || got["seat"] != nil {
|
||||
t.Errorf("%s names a seat: %v", id, got)
|
||||
}
|
||||
}
|
||||
// A reference in the form a registry seat answers is routed the same way, names the seat it came through (ENG-4), and its
|
||||
// bundle is fetched as an archive from the store.
|
||||
seatForm := mcat.Resolution{Node: "anchor", Modules: []mcat.Manifest{{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea", "image": "container-image-registry/gitea/server@" + digest},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "source": "oci-artifact-registry/gitea/config@" + digest}}}}}
|
||||
held.SeatHolder["container-image-registry"] = "distribution"
|
||||
seated, err := Declaration(seatForm, held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(seated, "gitea.server"); got["image"] != "anchor.internal:5101/gitea/server@"+digest || got["image-seat"] != "container-image-registry" || got["image-holder"] != "distribution" {
|
||||
t.Errorf("a seat-form image is %v, naming %v %v", got["image"], got["image-seat"], got["image-holder"])
|
||||
}
|
||||
if got := fileNamed(seated, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("a seat-form archive is fetched from %v", got)
|
||||
}
|
||||
held.SeatHolder = map[string]string{"mesh-artifact-store": "other-holder"}
|
||||
if held2, err := Declaration(r, held); err != nil || fileNamed(held2, "gitea.server")["image-holder"] != "other-holder" {
|
||||
t.Errorf("a new holder is not written: %v", err)
|
||||
}
|
||||
|
||||
// And the store moves: the same record, another address, without a rebuild.
|
||||
out, err = Declaration(r, mcat.Rendering{ArtifactStore: "laptop.internal:5000"})
|
||||
if err != nil {
|
||||
|
||||
@@ -102,7 +102,13 @@ func plainly(value any) string {
|
||||
// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR
|
||||
// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses
|
||||
// that already; this is the same rule where the file is made, for a manifest registered before it.
|
||||
func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error {
|
||||
//
|
||||
// **A key the binding does not say is refused with why the node-engine fills the file** (issue 1966): filledHere
|
||||
// (notHandedWhy). What a provider's helper answers reaches only a handed module's files, which the provisioner's client
|
||||
// fills; a file the client should fill (handedFile) refused here is a handed module not handed, and says so, never only
|
||||
// the keys it lacks.
|
||||
func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool,
|
||||
handedFile bool, filledHere string) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
@@ -135,10 +141,16 @@ func boundInto(resource map[string]any, known map[string]map[string]string, modu
|
||||
return token
|
||||
}
|
||||
value, said := values[key]
|
||||
if !said && handedFile {
|
||||
refused = fmt.Errorf(
|
||||
"handed module not handed: %s asks its %s binding for %q, which only the provisioner's client fills, "+
|
||||
"and %s (issue 1966)", module, provision, key, filledHere)
|
||||
return token
|
||||
}
|
||||
if !said {
|
||||
refused = fmt.Errorf(
|
||||
"%s asks its %s binding for %q, and what answers it says %s",
|
||||
module, provision, key, orNothing(namesOfKeys(values)))
|
||||
"%s asks its %s binding for %q, and what answers it says %s (%s)",
|
||||
module, provision, key, orNothing(namesOfKeys(values)), filledHere)
|
||||
return token
|
||||
}
|
||||
return value
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package resolution
|
||||
|
||||
import (
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// controlRecordsDecision is the record that made the two seats.
|
||||
const controlRecordsDecision = "novox/hq ADR 0565"
|
||||
|
||||
// wantsVerbs are the assigner's verbs on mesh-wants (ADR 0565 clauses 7, 8, 9, 15).
|
||||
func wantsVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "upgrade", Description: "Whether each module's new builds wait for a person: its policy (roll, together or record) " +
|
||||
"and where it comes from (a person, the module, the bus, its irreplaceable data, the default); the bus is never " +
|
||||
"rolled. With module, that one; with module, policy and why, a person's choice for it, recorded as a hand act; " +
|
||||
"with policy default, the choice taken back. With compare, each node's own composition and the assigner's findings.",
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module",
|
||||
"policy": "with module: roll, together, record, or default to take a person's choice back",
|
||||
"why": "with policy: why, required and kept",
|
||||
"compare": "\"true\": each node's own composition, by its digest, and the assigner's findings",
|
||||
}, nil, "compare")},
|
||||
{Name: "bus", Description: "What a bus upgrade would do: the fingerprint of the bus module each node runs against the " +
|
||||
"indexer's newest, and the last bus steps from the hand-act log and the assigner's puts of the bus. Read-only: " +
|
||||
"the step itself is the put on the operator's warrant.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
// handActsVerbs are mesh-hand-acts' verbs (ADR 0565 clause 10), as the controller's took them.
|
||||
func handActsVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "hand-act", Description: "Record one act done by hand: what, why, its cause in a word or a condition's kind " +
|
||||
"(the word a second act for the same reason uses) and the condition it addresses. Who did it is the caller the " +
|
||||
"runtime proves, never said by the caller. Said as act-recorded.",
|
||||
Input: schema(map[string]string{
|
||||
"what": "what was done, in a line",
|
||||
"why": "why it had to be done by hand",
|
||||
"cause": "the cause in a word, or a condition's kind",
|
||||
"condition": "the key of the condition it addressed, if any (optional)",
|
||||
"verb": "the act's verb in the log: a repair when absent, `hand-act drill` for a drill, or a recorder's own",
|
||||
}, []string{"what", "why"})},
|
||||
{Name: "hand-acts", Description: "What was done by hand lately, over this log and the controller's: who, why and the " +
|
||||
"cause of each, and the causes recorded twice within a fortnight (a healer is wanted).",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 14)", "cause": "only this cause",
|
||||
"who": "only acts whose who contains this", "warrants": "\"true\": every warrant recorded"}, nil, "warrants")},
|
||||
{Name: "warranted", Description: "Record what a module did on the operator's warrant: who chose, through which channel, " +
|
||||
"with which proofs and which answer read from the router's own record of that module's ask, never from the " +
|
||||
"caller; once per ask. Refused for an ask still open, ended without a choice, or not that module's.",
|
||||
Input: schema(map[string]string{"asker": "the module that asked", "ask": "its ask's id"}, []string{"asker", "ask"})},
|
||||
}
|
||||
}
|
||||
@@ -415,6 +415,11 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string]
|
||||
// family's member (ADR 0283), so the module reads its absence as not given.
|
||||
continue
|
||||
}
|
||||
if why := VaultDisagrees(with, m); sealed == "" && why != "" {
|
||||
// The vault left it out of what it answered this node, its plan handing it: never a secret to make.
|
||||
return nil, fmt.Errorf("handed module not handed: %s needs its own secret %q, which the vault answers "+
|
||||
"its provisioner's client alone, %s (issue 1966)", m.Module, name, why)
|
||||
}
|
||||
if sealed == "" {
|
||||
// Declared and not made. Refused rather than skipped: a module whose own
|
||||
// credential is silently absent starts, fails to authenticate, and the reason is
|
||||
@@ -912,7 +917,8 @@ func resolutionCompose(r mcat.Resolution, with mcat.Rendering, owner map[string]
|
||||
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
|
||||
// holds it in the clear — so there is nothing for the host to be the only witness of,
|
||||
// and sending it already filled in means the host learns no new field.
|
||||
if err := boundInto(copied, known, m.Module, absent); err != nil {
|
||||
filledHere, handedFile := notHandedWhy(with, m, unsettled)
|
||||
if err := boundInto(copied, known, m.Module, absent, handedFile, filledHere); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And what the module could not have written: the machine it turned out to be
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
package resolution
|
||||
|
||||
import (
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// engineSeat is the seat's row in the closed set.
|
||||
func engineSeat() mcat.Seat {
|
||||
return mcat.Seat{Name: mcat.EngineSeat, Scope: mcat.ScopeNode, Serves: mcat.EngineVerbs(), Emits: append([]string(nil), mcat.EngineEvents...),
|
||||
HeldByTheEngine: true, Decision: "novox/hq ADR 0563"}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
@@ -14,73 +13,6 @@ const filesAsAccount = "as the node's operator account, through sudo -n where it
|
||||
|
||||
const overrideWord = "a reason, to change a path a module owns anyway (optional)"
|
||||
|
||||
// filesVerbs is the contract every holder of node-files serves (novox/hq ADR 0394 §1): the reads of node-reads,
|
||||
// with their descriptions restated for the operator's account, and the writes.
|
||||
func filesVerbs() []mcat.Verb {
|
||||
var verbs []mcat.Verb
|
||||
for _, v := range readsVerbs() {
|
||||
v.Description = strings.Replace(v.Description,
|
||||
"Read as the holder's own account, which holds no secret: a component it may not reach says so.",
|
||||
"", 1)
|
||||
v.Description = strings.TrimSpace(v.Description) + " Read " + filesAsAccount + "."
|
||||
verbs = append(verbs, v)
|
||||
}
|
||||
write := func(name, description string, in map[string]string, required []string, optional []string, replaces ...string) mcat.Verb {
|
||||
in["reason"] = "why the file is changed, for the audit trail"
|
||||
in["override"] = overrideWord
|
||||
return mcat.Verb{Name: name, Description: description + " Refuses a path a module owns " +
|
||||
"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
|
||||
Input: schema(in, append(required, "reason"), append(optional, "override")...), Replaces: replaces}
|
||||
}
|
||||
return append(verbs,
|
||||
write("write", "Write one text file whole, replacing what it holds; the old content goes to the node's trash.",
|
||||
map[string]string{"path": "an absolute path", "content": "the text to write",
|
||||
"mode": "the file's mode, such as 0644 (optional)"},
|
||||
[]string{"path", "content"}, []string{"mode"}, "tee", ">"),
|
||||
write("copy", "Copy a file or directory to a path on this node. A target on another node (<node>:/path) is "+
|
||||
"refused: copying between nodes comes in step 2 (issue 903). An existing target is overwritten, the old one "+
|
||||
"kept in the node's trash for 7 days.",
|
||||
map[string]string{"from": "an absolute path", "to": "an absolute path on this node"},
|
||||
[]string{"from", "to"}, nil, "cp", "cp -a", "ssh <node> cp"),
|
||||
write("move", "Move or rename a file or directory. A target on another node is refused: copying between nodes "+
|
||||
"comes in step 2 (issue 903). An existing target is overwritten, the old one kept in the node's trash for 7 days.",
|
||||
map[string]string{"from": "an absolute path", "to": "an absolute path"},
|
||||
[]string{"from", "to"}, nil, "mv"),
|
||||
write("delete", "Delete a file or directory into the node's trash, where restore can bring it back.",
|
||||
map[string]string{"path": "an absolute path", "recursive": "\"true\": delete a directory with what it holds"},
|
||||
[]string{"path"}, []string{"recursive"}, "rm", "rm -r"),
|
||||
write("mkdir", "Make a directory.",
|
||||
map[string]string{"path": "an absolute path", "parents": "\"true\": make the missing parents too"},
|
||||
[]string{"path"}, []string{"parents"}, "mkdir", "mkdir -p"),
|
||||
write("chmod", "Change the mode of a file or directory.",
|
||||
map[string]string{"path": "an absolute path", "mode": "the mode, such as 0644 or u+x",
|
||||
"recursive": "\"true\": also everything under a directory"},
|
||||
[]string{"path", "mode"}, []string{"recursive"}, "chmod"),
|
||||
write("chown", "Change the owner and group of a file or directory.",
|
||||
map[string]string{"path": "an absolute path", "owner": "an account's name or number",
|
||||
"group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"},
|
||||
[]string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"), mcat.Verb{Name: "upload", Description: "Send one file of this node into a bucket of the object store, over a short signed link " +
|
||||
"and never over the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. " +
|
||||
"At most 100 MiB. A place a secret is kept, a known credential file, or a link to one is refused by name, " +
|
||||
"before anything is sent. Read " + filesAsAccount + ".",
|
||||
Input: schema(map[string]string{"path": "an absolute path to a file", "bucket": "the bucket the file goes into",
|
||||
"key": "the object's key (optional; the file's SHA-256 when not given)"}, []string{"path", "bucket"})}, mcat.
|
||||
// download is a plain Verb, not through write(): write() types its optional arguments as true/false
|
||||
// switches, which a hash and a reason are not (issue 1719).
|
||||
Verb{Name: "download", Description: "Write one object of a bucket to a file of this node, over a short signed link and " +
|
||||
"never over the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to " +
|
||||
"the key when the key is a SHA-256; otherwise no file is written. Refuses a path a module owns " +
|
||||
"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
|
||||
Input: schema(map[string]string{"bucket": "the bucket", "key": "the object's key", "path": "an absolute path",
|
||||
"sha256": "the SHA-256 the bytes must have (optional; the key when that is a SHA-256)",
|
||||
"reason": "why the file is changed, for the audit trail",
|
||||
"override": overrideWord}, []string{"bucket", "key", "path", "reason"})}, mcat.Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " +
|
||||
"original path and when it went.",
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}}, write("restore", "Put an item of the node's trash back where it was; refuses when something is there now.",
|
||||
map[string]string{"id": "an id from trash"}, []string{"id"}, nil, "trash-restore"),
|
||||
)
|
||||
}
|
||||
|
||||
// ownedPaths is every path a module owns on this node, as the holder is told (novox/hq ADR 0394 §5): the
|
||||
// paths every seat declares (ADR 0287) with the module holding the seat here, and the paths and files each
|
||||
// module's manifest places (declaredPaths). A seat's path wins where both name one.
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
package resolution
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// graphicalSessionSeats are the eleven, in the order ADR 0208's table reads, each with the verbs
|
||||
// research 026/05 starts it with. Three have none yet: the bar (which receives blocks instead,
|
||||
// ADR 0255), the compositor and the secret
|
||||
// service are roles a second holder competes for, and nothing has needed to ask them anything.
|
||||
func graphicalSessionSeats() []mcat.Seat {
|
||||
const decided = "novox/hq ADR 0208"
|
||||
return []mcat.Seat{
|
||||
{Name: mcat.LoginManagerSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "sessions", Description: "The sessions the login manager offers on this machine, and which " +
|
||||
"one the operator account starts by default.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}},
|
||||
{Name: mcat.DisplayServerSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "displays", Description: "The monitors connected now, each with its identity, its modes and " +
|
||||
"where it is placed; and the layout profile in force, if one matches.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
// Profiles are keyed by the monitors' identities and are the operator's data (ADR 0208 §6).
|
||||
{Name: "layout", Description: "The monitor layout profiles, keyed by the connected monitors' " +
|
||||
"identities: list them, save the current arrangement under a name, or apply one.",
|
||||
Input: withEnum(schema(map[string]string{
|
||||
"action": "list, save or apply",
|
||||
"name": "the profile to save or apply (save and apply only)",
|
||||
}, []string{"action"}), "action", "list", "save", "apply")},
|
||||
}},
|
||||
// It receives window-manager configuration lines from every other module (novox/hq ADR 0212):
|
||||
// bindings, start-up commands, rules — placed by the session's holder, never written into
|
||||
// its directory by the contributor.
|
||||
{Name: mcat.DisplaySessionSeat, Scope: mcat.ScopeNode, Decision: decided,
|
||||
Receives: []mcat.Receivable{{Kind: "config", Comment: "#"}}, Serves: []mcat.Verb{
|
||||
{Name: "reload", Description: "Reload the session's configuration in place, keeping its windows.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "workspaces", Description: "The session's workspaces: each one's name, output, and whether " +
|
||||
"it is visible or focused.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "windows", Description: "The session's windows: each one's title, class, workspace and " +
|
||||
"whether it has focus; narrowed to one workspace when named.",
|
||||
Input: schema(map[string]string{"workspace": "one workspace (optional)"}, nil)},
|
||||
}},
|
||||
{Name: mcat.TerminalEmulatorSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "open", Description: "Open a terminal window in the operator's session, running a command " +
|
||||
"or the login shell, in a directory or the account's home.",
|
||||
Input: schema(map[string]string{
|
||||
"command": "what to run in it (optional; the login shell when absent)",
|
||||
"directory": "where it starts (optional; the account's home when absent)",
|
||||
}, nil)},
|
||||
}},
|
||||
{Name: mcat.LauncherSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "menu", Description: "Put a menu of choices in front of the operator and answer with the " +
|
||||
"one chosen, or nothing when the menu was dismissed — the dmenu-compatible contract.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"choices"},
|
||||
"properties": map[string]any{
|
||||
"choices": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the lines to choose between, in order"},
|
||||
"prompt": map[string]any{"type": "string", "description": "what the menu asks (optional)"},
|
||||
}}},
|
||||
// A value the operator types and nobody sees (novox/hq ADR 0259 §10): a hidden prompt whose answer
|
||||
// is sealed to the asker's key, so it is never plaintext on the bus or in any call's record.
|
||||
// **Optional while its holders catch up** (ADR 0246): rofi serves it once this is live.
|
||||
{Name: "secret", Optional: true, Description: "Ask the operator for a value in a prompt that " +
|
||||
"does not show what is typed, and answer it sealed to the key the asker gives — never in " +
|
||||
"the clear — or cancelled when the prompt was dismissed or not answered in time.",
|
||||
// By name, never by words (the review of 2026-10-09, M4): the holder writes the prompt from the
|
||||
// module, the secret and the machine, and says the controller asks — the bus lets nobody else
|
||||
// ask it (broker.ControllerOnly) — so no caller puts words of its own before the operator.
|
||||
Input: schema(map[string]string{
|
||||
"module": "the module whose own secret is asked for",
|
||||
"secret": "the own secret's name",
|
||||
"node": "the machine the module runs on",
|
||||
"seal_to": "the asker's public sealing key: the answer is sealed to it",
|
||||
"timeout_seconds": "give up after this long (optional)",
|
||||
}, []string{"module", "secret", "node", "seal_to"})},
|
||||
}},
|
||||
{Name: mcat.NotifierSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "send", Description: "Show the operator a notification.",
|
||||
Input: withEnum(schema(map[string]string{
|
||||
"title": "the notification's summary",
|
||||
"body": "its text (optional)",
|
||||
"urgency": "low, normal (the default) or critical",
|
||||
}, []string{"title"}), "urgency", "low", "normal", "critical")},
|
||||
{Name: "history", Description: "The notifications shown lately, newest first.",
|
||||
Input: schema(map[string]string{"limit": "how many (optional, default 20)"}, nil)},
|
||||
}},
|
||||
{Name: mcat.LockScreenSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "lock", Description: "Lock the operator's session now.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}},
|
||||
{Name: mcat.ClipboardSeat, Scope: mcat.ScopeNode, Decision: decided, Serves: []mcat.Verb{
|
||||
{Name: "history", Description: "What the clipboard held lately, newest first.",
|
||||
Input: schema(map[string]string{"limit": "how many (optional, default 20)"}, nil)},
|
||||
{Name: "copy", Description: "Put text on the operator's clipboard.",
|
||||
Input: schema(map[string]string{"text": "the text"}, []string{"text"})},
|
||||
}},
|
||||
// It receives blocks as data (novox/hq ADR 0255): what a block shows, said for any bar, which
|
||||
// the holder renders in its own grammar. A module that knows something about the machine — the
|
||||
// power module its battery — gives the block; the bar knows no machine's hardware.
|
||||
{Name: mcat.BarSeat, Scope: mcat.ScopeNode, Decision: decided,
|
||||
Receives: []mcat.Receivable{{Kind: mcat.BarKindBlock, Comment: "#", Offered: true, Shape: barBlockShape()}}},
|
||||
{Name: mcat.CompositorSeat, Scope: mcat.ScopeNode, Decision: decided},
|
||||
{Name: mcat.SecretServiceSeat, Scope: mcat.ScopeNode, Decision: decided},
|
||||
}
|
||||
}
|
||||
|
||||
// withEnum narrows one string property of a schema to the values it may take, so a caller is told
|
||||
// the choices by the schema rather than by a refusal.
|
||||
func withEnum(s map[string]any, property string, values ...string) map[string]any {
|
||||
props := s["properties"].(map[string]any)
|
||||
p := props[property].(map[string]any)
|
||||
p["enum"] = values
|
||||
return s
|
||||
}
|
||||
|
||||
// barShows is what a contributed block may show, and the options each takes — the bar-neutral
|
||||
// vocabulary every holder of node-bar renders. A new entry is a decision, and every holder's template
|
||||
// renders it (the shape's examples) before it is accepted.
|
||||
var barShows = map[string]map[string]barOption{
|
||||
// The machine's battery: its charge, and how long it lasts or takes to fill.
|
||||
"battery": {"device": {kind: "text"}},
|
||||
// One line a command prints, run every interval seconds: what the bar cannot know itself.
|
||||
"command": {"command": {kind: "text", required: true}, "interval": {kind: "seconds", required: true}},
|
||||
// A value the contributor keeps on the bus (novox/hq ADR 0255): its state `<module>.<name>` and the
|
||||
// key, this machine's name when absent. The value is an object whose `text` is what is shown. The
|
||||
// holder is granted to read it and shows it however its bar shows a changing value; no file and
|
||||
// no command passes between the two modules.
|
||||
"state": {"state": {kind: "text", required: true}, "key": {kind: "text"}},
|
||||
}
|
||||
|
||||
// barBlockReads is the state and key a block shows, if any; an empty key is the machine's own.
|
||||
func barBlockReads(data map[string]any) []mcat.StateRead {
|
||||
if data["shows"] != "state" {
|
||||
return nil
|
||||
}
|
||||
options, _ := data["options"].(map[string]any)
|
||||
s, ok := options["state"].(string)
|
||||
if !ok || s == "" {
|
||||
return nil
|
||||
}
|
||||
key, _ := options["key"].(string)
|
||||
return []mcat.StateRead{{Read: s, Key: key}}
|
||||
}
|
||||
|
||||
type barOption struct {
|
||||
kind string // text, or seconds (a whole number from 1 to 86400)
|
||||
required bool
|
||||
}
|
||||
|
||||
// barBlockShape is the shape of a block (novox/hq ADR 0255): which bar, where on it, in what order,
|
||||
// what it shows, and that kind's options.
|
||||
//
|
||||
// - bar: top or bottom;
|
||||
// - place: resources (beside the processor, memory and disks) or status (beside the sound, before the
|
||||
// clock) — the two places every holder keeps for contributed blocks;
|
||||
// - order: 0–99 within the place, 50 when absent;
|
||||
// - shows: battery or command; options: what that kind takes.
|
||||
func barBlockShape() *mcat.Shape {
|
||||
return &mcat.Shape{
|
||||
Text: true,
|
||||
Places: []mcat.PlaceField{{Field: "bar", Values: []string{"bottom", "top"}},
|
||||
{Field: "place", Values: []string{"resources", "status"}}},
|
||||
Order: "order",
|
||||
Check: barBlockProblems,
|
||||
Reads: barBlockReads,
|
||||
Examples: []map[string]any{
|
||||
{"bar": "bottom", "place": "status", "shows": "battery"},
|
||||
{"bar": "bottom", "place": "status", "shows": "battery", "options": map[string]any{"device": "BAT0"}},
|
||||
{"bar": "bottom", "place": "status", "shows": "command",
|
||||
"options": map[string]any{"command": "echo \"a line\"", "interval": float64(5)}},
|
||||
{"bar": "bottom", "place": "status", "shows": "state", "options": map[string]any{"state": "power.draw"}},
|
||||
{"bar": "bottom", "place": "status", "shows": "state",
|
||||
"options": map[string]any{"state": "power.draw", "key": "a-machine"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func barBlockProblems(data map[string]any) []string {
|
||||
var problems []string
|
||||
for field := range data {
|
||||
if !oneOf([]string{"bar", "place", "order", "shows", "options"}, field) {
|
||||
problems = append(problems, fmt.Sprintf("a block has no field %q; it has bar, place, order, shows and options", field))
|
||||
}
|
||||
}
|
||||
shows, _ := data["shows"].(string)
|
||||
takes, known := barShows[shows]
|
||||
if !known {
|
||||
return append(problems, fmt.Sprintf("a block shows %v; it shows one of %s", data["shows"],
|
||||
strings.Join(sortedKeys(barShows), ", ")))
|
||||
}
|
||||
options := map[string]any{}
|
||||
if raw, has := data["options"]; has {
|
||||
o, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return append(problems, "a block's options are an object")
|
||||
}
|
||||
options = o
|
||||
}
|
||||
for name, v := range options {
|
||||
opt, ok := takes[name]
|
||||
if !ok {
|
||||
problems = append(problems, fmt.Sprintf("a %s block has no option %q; it takes %s", shows, name,
|
||||
orNothing(sortedKeys(takes))))
|
||||
continue
|
||||
}
|
||||
switch opt.kind {
|
||||
case "text":
|
||||
if t, isText := v.(string); !isText || strings.TrimSpace(t) == "" {
|
||||
problems = append(problems, fmt.Sprintf("a %s block's %s is text", shows, name))
|
||||
}
|
||||
case "seconds":
|
||||
if _, whole := wholeIn(v, 1, 86400); !whole {
|
||||
problems = append(problems, fmt.Sprintf("a %s block's %s is a whole number of seconds from 1 to 86400", shows, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(takes) {
|
||||
if _, has := options[name]; takes[name].required && !has {
|
||||
problems = append(problems, fmt.Sprintf("a %s block needs the option %s", shows, name))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -96,22 +96,78 @@ func renderingHanding(with mcat.Rendering, m mcat.Manifest) bool {
|
||||
}
|
||||
|
||||
// Handing is whether a machine composed with this rendering hands the module to the provisioner, its files and
|
||||
// its own secrets (design 57 P6, V4): a moved module there, assigned or only pulled in by another (issue 765), on a
|
||||
// machine that can take it. The controller makes and seals none of such a module's secrets for that machine: its
|
||||
// client asks the vault.
|
||||
// its own secrets (design 57 P6, V4): a moved module there whose build kept what the client fills from
|
||||
// (catalogue.Handed, written by Resolve), assigned or only pulled in by another (issue 765), on a machine that can take
|
||||
// it (its node-engine writes a filled file on the client's word, and the client's account is named). The controller
|
||||
// makes and seals none of such a module's secrets for that machine: its client asks the vault.
|
||||
//
|
||||
// **Where the vault judged the module, its word is the rule** (Rendering.Assigned, issue 774): the vault leaves a
|
||||
// module's own secrets out of `own` exactly when the plan in its assignment hands it (AssignmentHands), so a node
|
||||
// composing itself hands exactly those, whatever its own reading of the manifest and the machine says. On 2026-10-31
|
||||
// postgres on novox was held: its plan handed its secrets, the vault answered none of them, and the node, judging by
|
||||
// the manifest and the machine, composed them itself and refused for want of `reader`. Two readers, one rule.
|
||||
// **The manifest, its build and the machine are the rule, never the plan** (issue 1966). The plan in a module's
|
||||
// assignment is what a node's last composition made of it; when the vault's reading of it (Rendering.Assigned) was the
|
||||
// rule, a module with no plan yet was judged not handed, composed the old way, refused there (its files ask the
|
||||
// provider's helper for what only the helper answers), and so never got the plan that would hand it: on 2026-11-04
|
||||
// ace's three build modules and ssh-client, and shanks's mesh-build-image and ssh-client. The vault's word follows the
|
||||
// plans this rule makes, one composition later, and is read only to say where the two disagree (VaultDisagrees).
|
||||
func Handing(with mcat.Rendering, m mcat.Manifest) bool {
|
||||
if hands, judged := with.Assigned[m.Module]; judged {
|
||||
return hands
|
||||
}
|
||||
return m.Moved && m.Handed != nil && with.Hands && with.Provisioner != ""
|
||||
}
|
||||
|
||||
// machineWhy is why a machine does not take a moved module's handed files, "" where it does.
|
||||
func machineWhy(with mcat.Rendering) string {
|
||||
switch {
|
||||
case !with.Hands:
|
||||
return "this machine's node-engine has not stated that it writes a filled file on the provisioner client's word"
|
||||
case with.Provisioner == "":
|
||||
return "this machine's declaration names no account the provisioner's client runs as"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// VaultDisagrees says why this composition does not hand a module the vault judged handed (Rendering.Assigned: the plan
|
||||
// in its assignment hands it, so the vault left its own secrets out of what it answered this node), "" where they agree
|
||||
// or the vault said nothing of it. Said where one of those secrets is then missing, as a handed module not handed, never
|
||||
// as a secret to make (issue 1966).
|
||||
func VaultDisagrees(with mcat.Rendering, m mcat.Manifest) string {
|
||||
if !with.Assigned[m.Module] || Handing(with, m) {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case !m.Moved:
|
||||
return "the vault judged it handed by its plan, and it is not moved"
|
||||
case m.Handed == nil:
|
||||
return "the vault judged it handed by its plan, and its build kept no templates"
|
||||
}
|
||||
return "the vault judged it handed by its plan, and " + machineWhy(with)
|
||||
}
|
||||
|
||||
// notHandedWhy says why a file of this module is composed here, filled by the node-engine, rather than by the
|
||||
// provisioner's client: the module is not moved, its build kept nothing to hand, the machine cannot take it, its build
|
||||
// kept no template for this file, or the file merges the layers' settings (begin). Said where a placeholder only a
|
||||
// handed file can name is refused (boundInto); handedFile is whether the module's manifest and build hand this file, so
|
||||
// that a refusal of it is a handed module not handed.
|
||||
func notHandedWhy(with mcat.Rendering, m mcat.Manifest, r map[string]any) (why string, handedFile bool) {
|
||||
id := fmt.Sprint(r["id"])
|
||||
switch {
|
||||
case !m.Moved:
|
||||
return m.Module + " is not moved, so its node-engine fills its files", false
|
||||
case m.Handed == nil:
|
||||
return m.Module + " is moved and its build kept no templates, so its node-engine fills its files", false
|
||||
}
|
||||
if _, kept := m.Handed.Templates[id]; !kept {
|
||||
return fmt.Sprintf("its build kept no template for %s, so its node-engine fills that file", id), false
|
||||
}
|
||||
if mergesFile(r) {
|
||||
return fmt.Sprintf("%s merges the layers' settings, which its node-engine does (issue 831)", id), false
|
||||
}
|
||||
if why := machineWhy(with); why != "" {
|
||||
return why, true
|
||||
}
|
||||
// Every way a handed file is not handed is named above: the module moved with templates, a template for this file, no
|
||||
// merge, a machine that takes it. Reached, the composition and Handing disagree, and that is a fault in this code, said as
|
||||
// one so it is reported and not read as a reason about the module or the machine (issue 1966).
|
||||
return fmt.Sprintf("a fault in the node-engine's composition, to be reported: %s is moved, its build kept a template for %s, "+
|
||||
"and this machine takes handed files, yet the composition filled it itself", m.Module, id), true
|
||||
}
|
||||
|
||||
// requiredByWords begins why a module only pulled in by another is on a machine (Resolution.Because): `required by
|
||||
// <module>`.
|
||||
const requiredByWords = "required by "
|
||||
|
||||
@@ -113,9 +113,10 @@ func TestAHandedModulesLocalCredentialsOfTheVaultAreNamedByLocalInItsPlan(t *tes
|
||||
|
||||
// **Issue 969: a key its provider serves only at run time.** A moved module's file may name `${bound:…}` of a
|
||||
// provision whose provider tells it at run time (ssh-client's known_host of the forge). Where the machine takes
|
||||
// handed files the client fills it and the module composes; where it does not the controller must fill it from what
|
||||
// the provider serves statically, cannot, and refuses it with the very words an unmarked module is refused with
|
||||
// (ADR 0349 §17: the old way, byte for byte).
|
||||
// handed files the client fills it and the module composes, whatever the vault judged of a plan not made yet (issue
|
||||
// 1966); where it does not the node-engine must fill it from what the provider serves statically, cannot, and refuses
|
||||
// it as a handed module not handed, saying why the machine does not take it (issue 1966), where an unmarked module is
|
||||
// refused with the keys the binding says and that it is not moved.
|
||||
func TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded(t *testing.T) {
|
||||
text := "Host ${bound:postgres-database:at}\n KnownHost ${bound:postgres-database:known_host}\n"
|
||||
moved := strings.Replace(botMoved(), `"own-secrets":{"token":"${dir:state}/token"}`,
|
||||
@@ -130,17 +131,38 @@ func TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded(t *testi
|
||||
t.Fatalf("handed, the module is filled by the client and the controller names nothing: %v %s", c.Filled, got)
|
||||
}
|
||||
|
||||
for name, with := range map[string]mcat.Rendering{
|
||||
"an engine that does not take a filled file alone": {Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
|
||||
"no provisioner client on the machine": {Hands: true, ArtifactStore: "store:5000"},
|
||||
// Issue 1966: the vault judged it not handed (no plan yet); it is handed all the same.
|
||||
noPlan := handingTo("store:5000")
|
||||
noPlan.Assigned = map[string]bool{"bot": false}
|
||||
if c, err := Compose(withDatabase(t, moved, text, true), noPlan); err != nil || len(c.Filled) == 0 {
|
||||
t.Fatalf("with no plan yet, the module is not handed: %v", err)
|
||||
}
|
||||
|
||||
for name, with := range map[string]struct {
|
||||
with mcat.Rendering
|
||||
why string
|
||||
}{
|
||||
"an engine that does not take a filled file alone": {mcat.Rendering{Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000"},
|
||||
"has not stated that it writes a filled file"},
|
||||
"no provisioner client on the machine": {mcat.Rendering{Hands: true, ArtifactStore: "store:5000"},
|
||||
"names no account the provisioner's client runs as"},
|
||||
} {
|
||||
_, err := Compose(withDatabase(t, moved, text, true), sealedForBot(with))
|
||||
if err == nil || !strings.Contains(err.Error(), `asks its postgres-database binding for "known_host"`) {
|
||||
t.Fatalf("%s: the key is not refused: %v", name, err)
|
||||
_, err := Compose(withDatabase(t, moved, text, true), sealedForBot(with.with))
|
||||
if err == nil || !strings.Contains(err.Error(), `handed module not handed: bot asks its postgres-database binding for "known_host"`) ||
|
||||
!strings.Contains(err.Error(), with.why) {
|
||||
t.Fatalf("%s: the key is not refused as a handed module not handed: %v", name, err)
|
||||
}
|
||||
_, before := Compose(withDatabase(t, unmoved, text, false), sealedForBot(with))
|
||||
if before == nil || before.Error() != err.Error() {
|
||||
t.Fatalf("%s: refused otherwise than the unmarked module: %v / %v", name, err, before)
|
||||
_, before := Compose(withDatabase(t, unmoved, text, false), sealedForBot(with.with))
|
||||
if before == nil || !strings.Contains(before.Error(), `bot asks its postgres-database binding for "known_host", and what answers it says "as", "at" or "from" (bot is not moved`) {
|
||||
t.Fatalf("%s: the unmarked module is refused %v", name, before)
|
||||
}
|
||||
}
|
||||
|
||||
// And where nothing keeps it from being handed yet the composition filled the file itself, the reason says it is a fault
|
||||
// in this code (the review of the vault's judgement, issue 1966), never a reason about the module or the machine.
|
||||
whole := mcat.Manifest{Module: "bot", Moved: true, Handed: &mcat.Handed{Templates: map[string]string{"f": "templates/f"}}}
|
||||
if why, handedFile := notHandedWhy(handingTo("store:5000"), whole, map[string]any{"id": "f"}); !handedFile ||
|
||||
!strings.Contains(why, "a fault in the node-engine's composition") {
|
||||
t.Errorf("the impossible branch says %q (handed file %v)", why, handedFile)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package resolution
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -60,14 +61,25 @@ func TestAHandedModulesSecretsAreNamedAndNoValueIsSealed(t *testing.T) {
|
||||
if want := []mcat.PlanSecret{{Name: "token", Path: path}}; !reflect.DeepEqual(c.Plans["bot"].Secrets, want) {
|
||||
t.Fatalf("the plan names secrets %+v, want %+v", c.Plans["bot"].Secrets, want)
|
||||
}
|
||||
// Issue 774 (postgres on novox, 2026-10-31): where the vault judged it handed, its secrets are the vault's even on a
|
||||
// machine the node reads as not taking a filled file, and nothing is refused for a value the vault never answers.
|
||||
judged, err := Compose(r, mcat.Rendering{ArtifactStore: "store:5000", Assigned: map[string]bool{"bot": true}})
|
||||
// Issue 1966 (ace, 2026-11-04): the vault judged it not handed, its assignment carrying no plan yet; the manifest,
|
||||
// its build and the machine hand it all the same, so the plan that would hand it is made.
|
||||
withoutPlan := handingTo("store:5000")
|
||||
withoutPlan.Assigned = map[string]bool{"bot": false}
|
||||
first, err := Compose(r, withoutPlan)
|
||||
if err != nil {
|
||||
t.Fatalf("a module the vault judged handed is refused: %v", err)
|
||||
t.Fatalf("a module the vault judged not handed, with no plan yet, is refused: %v", err)
|
||||
}
|
||||
if want := []mcat.PlanSecret{{Name: "token", Path: path}}; !reflect.DeepEqual(judged.Plans["bot"].Secrets, want) {
|
||||
t.Errorf("judged handed by the vault, the plan names secrets %+v, want %+v", judged.Plans["bot"].Secrets, want)
|
||||
if want := []mcat.PlanSecret{{Name: "token", Path: path}}; !reflect.DeepEqual(first.Plans["bot"].Secrets, want) {
|
||||
t.Errorf("with no plan yet, the plan names secrets %+v, want %+v", first.Plans["bot"].Secrets, want)
|
||||
}
|
||||
// Issue 774 (postgres on novox, 2026-10-31), as issue 1966 leaves it: where the vault judged it handed and the machine
|
||||
// cannot take it, the vault answered none of its secrets, and the refusal says a handed module is not handed, never
|
||||
// a secret to make.
|
||||
_, err = Compose(r, mcat.Rendering{ArtifactStore: "store:5000", Assigned: map[string]bool{"bot": true}})
|
||||
var notMade *NotMadeError
|
||||
if err == nil || errors.As(err, ¬Made) || !strings.Contains(err.Error(), "handed module not handed: bot needs its own secret \"token\"") ||
|
||||
!strings.Contains(err.Error(), "has not stated that it writes a filled file") {
|
||||
t.Errorf("judged handed by the vault on a machine that cannot take it, the composition says %v", err)
|
||||
}
|
||||
// A process naming the token's file still restarts when its client writes it.
|
||||
for _, res := range c.Resources {
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package resolution
|
||||
|
||||
import (
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// healthDecision is the record that made the four seats.
|
||||
const healthDecision = "novox/hq ADR 0564"
|
||||
|
||||
// conditionsVerbs are the condition keeper's verbs (ADR 0564 clauses 3, 9, 12).
|
||||
func conditionsVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then oldest, " +
|
||||
"raised from the observers' findings (the watchman, the self-check, the healers, the assigner) and cleared when " +
|
||||
"the finding goes, never by hand. Given a key, that one whole with its evidence; with history, every transition " +
|
||||
"lately.",
|
||||
Input: schema(map[string]string{
|
||||
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe, mesh, delivery or module",
|
||||
"severity": "only urgent, or only warning",
|
||||
"machine": "only those about this machine",
|
||||
"history": "\"true\": every raising, change, silence and clearing lately, oldest first",
|
||||
"days": "with history: how many days back (default 7, at most 90)",
|
||||
}, nil, "history")},
|
||||
{Name: "silence", Description: "Stop one condition's messages for a while, a hand act with why: it stays open and in " +
|
||||
"status, and the silence is kept as its transition in history.",
|
||||
Input: schema(map[string]string{
|
||||
"key": "the condition's key",
|
||||
"for": "how long: 30m, 4h, 2d; at most 7d",
|
||||
"why": "why, required and recorded",
|
||||
"cause": "the cause in a word (the condition's kind when absent)",
|
||||
}, []string{"key", "for", "why"})},
|
||||
{Name: "resync", Description: "Read every observer's findings again and raise and clear what differs from the open " +
|
||||
"conditions: the recovery of a keeper that missed a change.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
// watchdogsVerbs are the watchman's verbs (ADR 0564 clauses 4, 5, 12).
|
||||
func watchdogsVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "signals", Description: "Every watchdog of the signals table the watchman keeps, each a key that expires unless " +
|
||||
"its signal renews it: its row, what renews it, its bound, the age of its newest signal, and the finding its " +
|
||||
"expiry wrote.",
|
||||
Input: schema(map[string]string{"row": "one row, e.g. S1"}, nil)},
|
||||
{Name: "stuck", Description: "Every want, build or step the watchman says stopped moving past its bound, with since " +
|
||||
"when and the words of its finding.",
|
||||
Input: schema(map[string]string{"machine": "only those on this machine"}, nil)},
|
||||
{Name: "presence", Description: "Every machine and tool runner the watchman sees connected, with its last word, and " +
|
||||
"those it says silent; a machine that said it sleeps is shown as sleeping, not silent.",
|
||||
Input: schema(map[string]string{"machine": "one machine"}, nil)},
|
||||
{Name: "dead-letters", Description: "Every message a consumer on the bus gave up on, kept in DEAD_LETTERS: whose " +
|
||||
"consumer, the subject, how often it was handed over and when it was given up, newest first. With id: that one " +
|
||||
"whole. With deliver or drop, a hand act with why.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a dead letter's id, as the list gives it: that one whole, with its message",
|
||||
"consumer": "a consumer's name, or <stream>.<consumer>: only what that one gave up on",
|
||||
"limit": "how many to list, newest first (default 50); only when listing",
|
||||
"deliver": "a dead letter's id: deliver it again to the consumer that gave it up (needs why)",
|
||||
"drop": "a dead letter's id: drop it for good (needs why)",
|
||||
"why": "with deliver or drop: why, required and recorded",
|
||||
"cause": "with deliver or drop: the cause in a word (dead-letter when absent)",
|
||||
}, nil)},
|
||||
{Name: "resync", Description: "Rebuild presence from the bus's connections, and the pulses and findings from the keys " +
|
||||
"the watchman watches, and take every dead-letter notice waiting: the recovery of a watchman that missed a change.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
// selfCheckVerbs are the self-check's verbs (ADR 0564 clause 6).
|
||||
func selfCheckVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "doctor", Description: "The self-check: the last run's verdict at once, each probe passed, failed, deferred " +
|
||||
"(its facts not on the bus yet, named) or could not run, and how long ago. With run, a run now; with probe, one " +
|
||||
"probe's verdict whole. Runs every five minutes on its own; each failure is a finding.",
|
||||
Input: schema(map[string]string{
|
||||
"run": "\"true\": run every probe now and answer the verdict",
|
||||
"probe": "one probe's id (D6): its verdict alone, with every finding",
|
||||
}, nil, "run")},
|
||||
{Name: "probes", Description: "The registry: what each probe asserts, where it reads its facts, and the finding it writes.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
// healersVerbs are the healers' verbs (ADR 0564 clauses 7, 8).
|
||||
func healersVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "healers", Description: "The healers: each registered response to one kind of condition, its repair, its " +
|
||||
"budget and what happens when it is spent; every act taken lately with its outcome, and the mesh-wide brake.",
|
||||
Input: schema(map[string]string{"days": "how many days of acts back (default 7)"}, nil)},
|
||||
{Name: "heal", Description: "Take one healer's act for one open condition now, within its budget and the brake.",
|
||||
Input: schema(map[string]string{
|
||||
"key": "the condition's key",
|
||||
"why": "why now, kept with the act",
|
||||
}, []string{"key", "why"})},
|
||||
{Name: "drill", Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with " +
|
||||
"what it tests. A drill is never a repair: no healer is wanted for it however often it is run.",
|
||||
Input: schema(map[string]string{
|
||||
"what": "what was done on purpose, in a line",
|
||||
"why": "what the drill tests",
|
||||
"condition": "the key of the condition the drill is meant to raise, if any (optional)",
|
||||
}, []string{"what", "why"})},
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,12 @@ func busSeatDelivering(t *testing.T, delivers string) {
|
||||
|
||||
func oldBroker() mcat.Manifest {
|
||||
return mcat.Manifest{Module: "old-broker", Provides: []mcat.Offer{{Name: "mesh-bus", Scope: mcat.ScopeMesh}},
|
||||
Claims: []mcat.Claim{{Name: "mesh-broker", Scope: mcat.ScopeMesh}}}
|
||||
Claims: []mcat.Claim{{Name: "mesh-broker", Scope: mcat.ScopeMesh, Serves: brokerVerbs()}}}
|
||||
}
|
||||
|
||||
func newBroker() mcat.Manifest {
|
||||
return mcat.Manifest{Module: "new-broker", Provides: []mcat.Offer{{Name: "mesh-bus", Scope: mcat.ScopeMesh}},
|
||||
Claims: []mcat.Claim{{Name: "mesh-broker", Scope: mcat.ScopeMesh}}}
|
||||
Claims: []mcat.Claim{{Name: "mesh-broker", Scope: mcat.ScopeMesh, Serves: brokerVerbs()}}}
|
||||
}
|
||||
|
||||
// Nothing on record: exactly the old rule. One claimant holds; two are refused.
|
||||
@@ -108,3 +108,16 @@ func TestARefusalWithNothingOnRecordNamesTheHandover(t *testing.T) {
|
||||
t.Fatalf("the refusal does not name the handover that records the holder: %s", problems[0])
|
||||
}
|
||||
}
|
||||
|
||||
// brokerVerbs are the verbs the seat mesh-broker promises its holder, as the contracts define them: a holder names every one
|
||||
// (ADR 0132), so a fixture holder does (issue 1966 ENG-3).
|
||||
func brokerVerbs() []string {
|
||||
seat, _ := mcat.SeatNamed("mesh-broker")
|
||||
var out []string
|
||||
for _, v := range seat.Serves {
|
||||
if !v.Optional {
|
||||
out = append(out, v.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -10,64 +10,6 @@ import (
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// readsVerbs is the contract every holder of node-reads serves (novox/hq ADR 0310 §2). The bounds are the
|
||||
// build's to set and each verb's description to state (ADR 0310, Consequences); the holder states them in
|
||||
// every answer it cuts.
|
||||
func readsVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "stat", Description: "Each component of one path or several, from / down: its type, owner and group " +
|
||||
"(name and number), mode, whether it is a link and to what, and whether it is a mount point and of which " +
|
||||
"file system. A link is not followed unless follow is \"true\". Read as the holder's own account, which " +
|
||||
"holds no secret: a component it may not reach says so. At most 200 paths in one call.",
|
||||
Input: schema(map[string]string{
|
||||
"paths": "one absolute path, or several separated by line breaks",
|
||||
"follow": "\"true\": also say where the path leads once every link is followed",
|
||||
}, []string{"paths"}, "follow"),
|
||||
Replaces: []string{"stat", "ls -ld", "namei -l", "findmnt -T"}},
|
||||
{Name: "readlink", Description: "What one link points to, as written, and where the path leads once every " +
|
||||
"link is followed, and whether that exists.",
|
||||
Input: schema(map[string]string{"path": "an absolute path"}, []string{"path"}),
|
||||
Replaces: []string{"readlink", "readlink -f", "realpath"}},
|
||||
{Name: "list", Description: "Every entry of one directory with its type, owner, group, mode, size and, for a " +
|
||||
"link, its target. At most 2000 entries; a longer directory is cut and says so. A place a secret is kept " +
|
||||
"is refused by name.",
|
||||
Input: schema(map[string]string{"path": "an absolute path to a directory"}, []string{"path"}),
|
||||
Replaces: []string{"ls", "ls -la"}},
|
||||
{Name: "read", Description: "One text file, or a part of it by line: at most 256 KiB in one answer " +
|
||||
"(64 KiB unless more is asked), cut at a line and said so. A binary file is refused; a place a secret is " +
|
||||
"kept, or a known credential file, is refused by name; a line shaped like a secret is withheld and its " +
|
||||
"number said. /proc and /sys are read as far as the holder's account may.",
|
||||
Input: schema(map[string]string{
|
||||
"path": "an absolute path to a file",
|
||||
"from": "the first line to answer, counted from 1 (optional)",
|
||||
"lines": "how many lines to answer (optional)",
|
||||
"bytes": "the most bytes to answer, up to 262144 (optional, 65536 unless given)",
|
||||
}, []string{"path"}),
|
||||
Replaces: []string{"cat", "head", "tail", "less"}},
|
||||
{Name: "search", Description: "Find a fixed string (never a pattern) in one file or under one directory: " +
|
||||
"every matching line with its file and number. Bounded: at most 500 matches (100 unless more is asked), " +
|
||||
"8 directories deep (4 unless more is asked), 5000 files and 4 MiB per file; links are not followed and " +
|
||||
"binary files are skipped. A place a secret is kept is skipped and named; a matching line shaped like a " +
|
||||
"secret is withheld. The answer says which bound it met.",
|
||||
Input: schema(map[string]string{
|
||||
"text": "the fixed string to find",
|
||||
"path": "an absolute path to a file or a directory",
|
||||
"depth": "how many directories deep, up to 8 (optional)",
|
||||
"matches": "the most matches to answer, up to 500 (optional)",
|
||||
"case": "\"false\": match whatever the case (optional)",
|
||||
}, []string{"text", "path"}, "case"),
|
||||
Replaces: []string{"grep", "grep -r", "grep -F"}},
|
||||
{Name: "account", Description: "One account or group by name or number, as this node's account files say: " +
|
||||
"an account's number, group, home, shell and every group it is in; a group's number and members. With " +
|
||||
"neither, every account and group. Never a password field.",
|
||||
Input: schema(map[string]string{
|
||||
"name": "an account's name or number (optional)",
|
||||
"group": "a group's name or number (optional)",
|
||||
}, nil),
|
||||
Replaces: []string{"getent passwd", "getent group", "id", "groups"}},
|
||||
}
|
||||
}
|
||||
|
||||
// secretPlaces is every place on this node a secret is kept, as the placed manifests say (novox/hq ADR 0310
|
||||
// §5): the directory of every module's own secret, every credential file a module is given and every grant
|
||||
// directory a provider is handed its consumers' credentials in, the place a module keeps the operator-sealed
|
||||
|
||||
@@ -157,6 +157,18 @@ func TestADirectlyAssignedModuleTheMachineCannotHostIsReportedNotRefused(t *test
|
||||
if !strings.Contains(WrongMachine("xorg", "seat", "server"), "wrong machine") {
|
||||
t.Errorf("the report reads like a missing module")
|
||||
}
|
||||
|
||||
// And a module whose calls only a tool runner stating them answers (ENG-2, issue 1966) is held off a machine whose record
|
||||
// does not say them, naming the capability; on one that does, it is hosted.
|
||||
withCalls := mcat.Node{Name: "server", Capabilities: map[string]bool{"container-runtime": true, "ledger-calls": true}}
|
||||
ledgerOwner := shelf(mod("records", nil, nil, []string{"ledger-calls"}))
|
||||
held, err := Resolve(ledgerOwner, []string{"records"}, server, mcat.World{})
|
||||
if err != nil || len(held.Unhostable) != 1 || len(held.Unhostable[0].Missing) != 1 || held.Unhostable[0].Missing[0] != "ledger-calls" {
|
||||
t.Errorf("a ledger owner on a machine whose runner states no ledger calls: %+v, %v", held.Unhostable, err)
|
||||
}
|
||||
if got, err := Resolve(ledgerOwner, []string{"records"}, withCalls, mcat.World{}); err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
|
||||
t.Errorf("a ledger owner on a machine whose runner states them was held: %v %v", got.Unhostable, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnhostableModuleSomethingRequiresRefusesTheNode(t *testing.T) {
|
||||
|
||||
@@ -33,7 +33,10 @@ var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
|
||||
var ofSeatReach = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):reach\}`)
|
||||
|
||||
// reachedSeats is every seat ${seat:…:reach} answers for.
|
||||
var reachedSeats = map[string]bool{"mesh-artifact-store": true}
|
||||
//
|
||||
// Both names of the store are answered with one address (ADR 0605, issue 1966 ENG-3): the seat it was held through and the
|
||||
// seat the registries' holder serves the same port as.
|
||||
var reachedSeats = map[string]bool{"mesh-artifact-store": true, "container-image-registry": true}
|
||||
|
||||
// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
|
||||
// holder — in a file's content, and in a value of a container's or a process's environment. The
|
||||
|
||||
@@ -110,6 +110,17 @@ func TestASeatsReachIsWhereThisMachineReachesItsHolder(t *testing.T) {
|
||||
if got := process["env"].(map[string]any)["REGISTRY"]; got != "anchor.internal:5100" {
|
||||
t.Fatalf("an environment value was filled with %q", got)
|
||||
}
|
||||
|
||||
// And the seat the registries' holder serves the same port through answers the same address (ENG-3).
|
||||
both := map[string]any{"type": "process", "name": "p", "env": map[string]any{
|
||||
"OLD": "${seat:mesh-artifact-store:reach}", "NEW": "${seat:container-image-registry:reach}"}}
|
||||
with.SeatReach["container-image-registry"] = "anchor.internal:5100"
|
||||
if err := seatInto(both, "x", with); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env := both["env"].(map[string]any); env["OLD"] != "anchor.internal:5100" || env["NEW"] != "anchor.internal:5100" {
|
||||
t.Fatalf("the two names of the store answer %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
// With no holder reachable, the runtime is not told to trust "": the member is dropped, and the
|
||||
|
||||
@@ -4,408 +4,10 @@ import (
|
||||
mcat "git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// defaultSeats is the set the mesh ships with — the seed for the control plane's seat table and the
|
||||
// fallback when it has none (novox/hq ADR 0122). It is the one place the closed set 0110 defines is
|
||||
// written; the store's table is seeded from it and thereafter is the live, editable copy.
|
||||
//
|
||||
// In the order a person reads it: the mesh's own, then a node's.
|
||||
var defaultSeats = append([]mcat.Seat{
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
{Name: "mesh-store", Scope: mcat.ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "databases", Description: "Every database the store holds, with its on-disk size.",
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"psql -l"}},
|
||||
{Name: "query", Description: "One read-only statement against one database the store holds.",
|
||||
Input: schema(map[string]string{"database": "the database to query", "sql": "the read-only statement"},
|
||||
[]string{"database", "sql"}), Replaces: []string{"psql"}},
|
||||
}},
|
||||
// **Delivers the mesh's own bus, not `amqp`.** Those were the same word until
|
||||
// ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is
|
||||
// the mesh's own transport. ADR 0128 then made that connection something a module requires
|
||||
// rather than receives ambiently — 23 of the catalogue's modules never speak, and an ambient
|
||||
// connection would mint a credential for each.
|
||||
{Name: mcat.BrokerSeat, Scope: mcat.ScopeMesh, Delivers: "mesh-bus", Decision: "novox/hq ADR 0079"},
|
||||
// The vault: the controller seals every minted credential with what it provides, which is the
|
||||
// test for a seat of the mesh's own (novox/hq ADR 0161) — a second provider of `secret` is a
|
||||
// second claimant, refused by name, rather than a candidate for a pin.
|
||||
//
|
||||
// Its verb `get` hands a secret on request and reply, sealed to the key the asker sends and only to an
|
||||
// asker the vault allows (novox/hq ADR 0349 §11, issue 674). `accept`, `secret-ask`, `issued` and `rotate`
|
||||
// take a value in and make one anew (ADR 0349 clauses 8 to 10, issue 675): the first three in two steps from
|
||||
// the controller alone, which carries the value sealed to a key the vault made for the call and cannot open
|
||||
// it. All optional until the vault that runs serves them (design 33 §7).
|
||||
{Name: "mesh-vault", Scope: mcat.ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "get", Optional: true, Description: "One secret, sealed to the public key the asker sends (the " +
|
||||
"sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the " +
|
||||
"bus proved it. Never the value in the clear, and never to a private key.",
|
||||
Input: schema(map[string]string{"name": "the secret's name",
|
||||
"public_key": "the asker's X25519 public key as SPKI PEM — never a private key"},
|
||||
[]string{"name", "public_key"})},
|
||||
{Name: "accept", Optional: true, Description: "Take a value given at the controller's terminal, from the " +
|
||||
"controller alone: name the secret and get a call and a key made for it, then send the call and the " +
|
||||
"value sealed to that key. The vault alone opens it, and never remakes an accepted credential.",
|
||||
Input: schema(vaultIntake(map[string]string{"node": "the machine, for a module's own secret",
|
||||
"module": "the module, for a module's own secret", "name": "the secret's name in the module",
|
||||
"provider": "the provider's machine, for a provision's credential", "provision": "the provision",
|
||||
"by": "who gave it at the terminal",
|
||||
"origin": "issued for a bus account the controller minted for a core or moved module and hands " +
|
||||
"at once (ADR 0374, issue 787); left out for a value given at the terminal"}), nil)},
|
||||
{Name: "secret-ask", Optional: true, Description: "Ask the operator for a module's own secret at a desk, " +
|
||||
"from the controller or a caller the vault's manifest names, named as the bus proved it (novox/hq ADR " +
|
||||
"0563 §8): the ask is recorded in the vault's store and bounded before the prompt " +
|
||||
"opens, and what is typed is sealed to a key the vault made for the call and opens there alone.",
|
||||
Input: schema(vaultIntake(map[string]string{"node": "the machine the secret is for",
|
||||
"module": "the module", "name": "the secret's name in the module",
|
||||
"desk": "the machine whose prompt asks", "asked_by": "who asked, as the bus named its caller",
|
||||
"outcome": "instead of sealed: how the prompt ended"}), nil)},
|
||||
{Name: "issued", Optional: true, Description: "Take the credential a provider's software issued for a " +
|
||||
"provision, from the controller alone, which asks the provider's tool with a key the vault made for " +
|
||||
"the call; kept as the provision's one credential, or, when the tool answers a value per holder named, " +
|
||||
"each as that holder's own; the same value again writes nothing.",
|
||||
Input: withHolders(schema(vaultIntake(map[string]string{"provision": "the provision",
|
||||
"provider": "the provider's machine", "module": "the provider's module"}), nil))},
|
||||
{Name: "rotate", Optional: true, Description: "Make a secret anew. Refused for a value a provider's " +
|
||||
"software issued and for an accepted credential. Never answers the value.",
|
||||
Input: schema(map[string]string{"node": "the machine, for a module's own secret", "module": "the module",
|
||||
"name": "the secret's name in the module", "provider": "the provider's machine",
|
||||
"provision": "the provision", "by": "who asked, when the controller carries the act"}, nil)},
|
||||
// The move of the controller's secrets (design 57's V3, issue 676): each machine's node-engine hands in what
|
||||
// was sealed to its machine, which the controller cannot open; the controller copies its record of asks.
|
||||
{Name: "move-in", Optional: true, Description: "Take one value of the controller's store in, as it is, " +
|
||||
"from the node-engine of a machine it was sealed to and no other caller: name the row, its origin " +
|
||||
"and generation and get a call and a key made for it, then send the call, the value sealed to that " +
|
||||
"key and a key to read back to. Never replaces a value the vault holds; answers what it kept, read " +
|
||||
"back sealed to that key.",
|
||||
Input: schema(vaultIntake(map[string]string{"node": "a module's own secret: its machine",
|
||||
"module": "its module", "name": "its name in the module",
|
||||
"provider": "a pair credential: the provider's machine", "provision": "the provision",
|
||||
"consumer": "the consumer's machine", "consumer_module": "the consuming module",
|
||||
"local": "the credential's local name, when it has one",
|
||||
"origin": "made, accepted or issued, as the controller's store says",
|
||||
"generation": "the controller's generation for the value", "made_at": "when it was made (RFC 3339)",
|
||||
"read_back_to": "the second step: a raw X25519 public key the read-back is sealed to"}), nil)},
|
||||
// A machine's node-engine states its own sealing key (novox/hq ADR 0372, issue 786).
|
||||
{Name: "node-key", Optional: true, Description: "State this machine's sealing key to the vault, in its " +
|
||||
"own name, from the node-engine of the machine alone: at its start and whenever the key changes. A " +
|
||||
"first or changed key is recorded and the machine's core secrets are said changed; the same key " +
|
||||
"again changes nothing.",
|
||||
Input: schema(map[string]string{"key": "the machine's sealing key, a raw X25519 public key in base64"}, nil)},
|
||||
// A machine's node-engine composes its own modules and asks their secrets sealed to its key (novox/hq ADR 0547
|
||||
// §2, ADR 0550 §5, issue 1126); granted as buscompose's Principal.AsksItsOwnSecrets says.
|
||||
{Name: "own", Optional: true, Description: "Answer the node-engine of one machine, about its own machine " +
|
||||
"alone (as the bus proved the caller, never an argument), its modules' own secrets and the pair " +
|
||||
"credentials it consumes or provides that it names, each sealed to the key it stated with node-key. A " +
|
||||
"credential the vault does not hold is named missing; never a value in the clear.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"credentials": map[string]any{
|
||||
"type": "array", "items": map[string]any{"type": "object"},
|
||||
"description": "the pair credentials the machine's composition reads, each {role: consumer|provider, " +
|
||||
"provision, consumer, consumer_module, provider, local}"}}}},
|
||||
{Name: "move-asks", Optional: true, Description: "Copy rows of the controller's record of asks at a desk " +
|
||||
"into the vault's, as they are and once, from the controller alone. No value is in any of them.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"asks": map[string]any{
|
||||
"type": "array", "items": map[string]any{"type": "object"},
|
||||
"description": "the controller's rows: id, node, module, name, desk, asked_by, opened_at, ended_at, outcome"}}}},
|
||||
{Name: "issue-account", Optional: true, Description: "Mint one bus account the grants part wants, now (novox/hq " +
|
||||
"ADR 0561 §9): only while the controller's lease is handed away, never a node's account or the " +
|
||||
"controller's, never anew for one the vault holds a value or an adopted hash for. Never the value.",
|
||||
Input: schema(map[string]string{"user": "the bus user, e.g. ace.mesh-build-go"}, []string{"user"})},
|
||||
{Name: "sign", Optional: true, Description: "Sign with the vault's key or authority (novox/hq ADR 0550 §2): " +
|
||||
"kind node-certificate signs the certificate of a machine's internal name, asked by that machine's " +
|
||||
"node-engine for the serving key its record states, built from bus state alone, or by mesh-enrolment for " +
|
||||
"a node it enrols, for the serving key the node presented at redeem (ADR 0587 §3); kind enrolment signs a " +
|
||||
"redeem answer under its own context, for mesh-enrolment alone. Answers the certificate or the signature " +
|
||||
"and the signer's fingerprint, and says signed; never a key.",
|
||||
Input: schema(map[string]string{"kind": "node-certificate or enrolment",
|
||||
"node": "the machine, for a node certificate: the caller's own, or the node mesh-enrolment enrols",
|
||||
"serving_key": "from mesh-enrolment: the serving key's public half the enrolling node presented",
|
||||
"reply": "the redeem answer's bytes in base64, for an enrolment"}, []string{"kind"})},
|
||||
// Enrolment (novox/hq ADR 0561 §5, ADR 0587 §3, §5; issue 774): mesh-enrolment is the only caller the vault's
|
||||
// manifest names for these two.
|
||||
{Name: "enrolment-token", Optional: true, Description: "Make a one-time token for a node mesh-enrolment " +
|
||||
"issues on the operator's warrant: its secret kept with its copy sealed to the operator key and its hash " +
|
||||
"written as the bus user enrol.<node>. Answers only what the joining machine must trust (the bus's " +
|
||||
"address, the bus certificate's fingerprint, the vault's public signing key), never the token; with " +
|
||||
"sealed, the token sealed to the operator key, for tokens to relay; with drop, the token and its hash " +
|
||||
"dropped, at the enrolment's end. Asked by mesh-enrolment alone, only from the lease write. The token " +
|
||||
"carries the first tunnel mesh-enrolment built at issue (ADR 0587 §3, insight of 2026-11-02).",
|
||||
Input: withObject(schema(map[string]string{"node": "the node the token is for",
|
||||
"expires": "when the token expires (RFC 3339), as the enrolment's key says",
|
||||
"sealed": "\"true\": answer the token sealed to the operator key",
|
||||
"drop": "\"true\": drop the node's token and its hash"}, []string{"node"}, "sealed", "drop"),
|
||||
"tunnel", "the node's first tunnel, built by mesh-enrolment at issue and carried in the token: the hub's "+
|
||||
"public key, the hub's endpoint, the node's tunnel address, its own tunnel key and the network's range",
|
||||
map[string]string{"hub_key": "the hub's tunnel public key", "hub_endpoint": "the hub's endpoint, host:port",
|
||||
"address": "the joining node's tunnel address",
|
||||
"key": "the joining machine's own tunnel public key, as given at issue",
|
||||
"range": "the private network's range, a prefix"})},
|
||||
{Name: "node-account", Optional: true, Description: "Mint a joining node's bus account node.<node>, sealed " +
|
||||
"to the sealing key it presented, and write its hash; asked again for the same node and keys, answers the " +
|
||||
"account it already minted; with retire, the account it minted for a revoked enrolment retired. Writes the " +
|
||||
"node's node-logins key naming node.<node>. Asked by mesh-enrolment alone. Never the value in the clear.",
|
||||
Input: schema(map[string]string{"node": "the joining node",
|
||||
"sealing_key": "the sealing key's public half the node presented, which the account is sealed to",
|
||||
"keys_digest": "the digest of every public key the node presented (delivery.KeysDigest)",
|
||||
"retire": "\"true\": retire the account minted for this node"}, []string{"node"}, "retire")},
|
||||
// A node's account rotates over two logins (novox/hq ADR 0583 §2-§5; issue 774), one node at a time, the bus
|
||||
// node last; the state node-logins says where each node stands.
|
||||
{Name: "rotate-node", Optional: true, Description: "Begin a node's account rotation to its other login: " +
|
||||
"answered only for a call carrying a hand act recorded through mesh-hand-acts, which the vault reads back by " +
|
||||
"its id. Makes the other login's value sealed to that node, writes it rotating in node-logins and says " +
|
||||
"rotation-begun; says node-account-ready once the bus node's have shows a list holding both logins. One " +
|
||||
"node at a time across the mesh, the bus node last: a call while a rotation runs waits until it is done.",
|
||||
Input: schema(map[string]string{"node": "the node whose account rotates", "why": "why, in a sentence",
|
||||
"act": "the id of the hand act recorded for it in mesh-hand-acts"}, []string{"node", "why", "act"})},
|
||||
{Name: "node-account-confirm", Optional: true, Description: "Confirm a node's switch to its new login: " +
|
||||
"asked by the node-engine dialled as that login, the runtime's proof of the caller being the confirmation. " +
|
||||
"The old login's hash is then removed, and rotation-done said once the bus node's have shows the list " +
|
||||
"without it.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "abandon", Optional: true, Description: "Abandon a node's rotation before its old login is retired: " +
|
||||
"the new login and its value dropped, the old one kept; a node that switched switches back first.",
|
||||
Input: schema(map[string]string{"node": "the node whose rotation is abandoned"}, []string{"node"})},
|
||||
}},
|
||||
// The owner of every config value (novox/hq ADR 0348 §9, design 57 K3): its holder alone writes the buckets
|
||||
// mesh-config_values, _resolved, _predefined and _proposals, and serves the one verb `config` (§1, §10).
|
||||
// Optional until the mesh-config that runs serves it (design 33 §7).
|
||||
{Name: "mesh-config", Scope: mcat.ScopeMesh, Decision: "novox/hq ADR 0348, ADR 0565",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "config", Optional: true, Description: "Set, clear, list or read the history of config values: " +
|
||||
"mesh config or node config for a predefined key, or a module's own value on one node, each set " +
|
||||
"with why, validated, and naming each module on each node whose value moves.",
|
||||
Input: configInput()},
|
||||
}},
|
||||
// The provisioner's server (novox/hq ADR 0349 clause 3, design 57 P4): its holder keeps the registry of which
|
||||
// domain answers which kind of placeholder, and the buckets mesh-provisioner_needs, _applied and _collect,
|
||||
// which it and each machine's provisioner client (for its own machine's keys) alone write. Its one verb reads
|
||||
// them. Optional until the mesh-provisioner that runs serves it (design 33 §7).
|
||||
{Name: "mesh-provisioner", Scope: mcat.ScopeMesh, Decision: "novox/hq ADR 0349",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "provisioning", Optional: true, Description: "Read what the provisioner keeps: the registry " +
|
||||
"(which domain answers which kind of placeholder), the needs each module on each node recorded, what " +
|
||||
"is applied where, or the items of every collection, narrowed by node and module. Reads only.",
|
||||
Input: schema(map[string]string{"what": "registry (when not given), needs, applied or collect",
|
||||
"node": "a node", "module": "a module: for collect, a contributor", "collection": "for collect: a collection"}, nil)},
|
||||
}},
|
||||
// Named for its scope since 2026-09-30 (novox/hq ADR 0156); `the-artifact-store` resolves to it as
|
||||
// an alias on a mesh that predates the rename. It serves artifacts of every kind a build makes —
|
||||
// images and archives, by digest — which is why the provision is the artifact store and not an
|
||||
// image registry.
|
||||
{Name: "mesh-artifact-store", Scope: mcat.ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"},
|
||||
// The catalog (novox/hq ADR 0604): the SDK's entry with its four verbs, so the copies of the table agree.
|
||||
sdkSeat(mcat.CatalogSeat),
|
||||
// The delivery's owner (novox/hq ADR 0239): one commit's journey from its pull request's head to every
|
||||
// machine, and a group of them, recorded and ordered by its holder, which asks the controller for each act.
|
||||
// What it says it says as its module (`mesh-delivery.transition`), through the runtime that launches it.
|
||||
{Name: mcat.DeliverySeat, Scope: mcat.ScopeMesh, Serves: deliveryVerbs(), Decision: "novox/hq ADR 0239"},
|
||||
// Placement (novox/hq ADR 0567): which module runs on which node, state on the bus held by the assigner, changed only by
|
||||
// a proposal the nodes answer and the operator approves. Its states and events are the holder module's.
|
||||
{Name: mcat.PlacementSeat, Scope: mcat.ScopeMesh, Serves: placementVerbs(), Decision: "novox/hq ADR 0567"},
|
||||
// The delivery's checker (novox/hq ADR 0547 §10, issue 774): the try is its verb, not the controller's.
|
||||
{Name: mcat.CheckerSeat, Scope: mcat.ScopeMesh, Serves: checkerVerbs(), Decision: "novox/hq ADR 0547"},
|
||||
// The mesh's health, out of the controller (novox/hq ADR 0564 clause 1, health_seats.go): the condition keeper, the
|
||||
// watchman's watchdogs, the self-check and the healers.
|
||||
{Name: mcat.ConditionsSeat, Scope: mcat.ScopeMesh, Decision: healthDecision,
|
||||
Emits: []string{"condition-raised", "condition-changed", "condition-cleared"}, Serves: conditionsVerbs()},
|
||||
{Name: mcat.WatchdogsSeat, Scope: mcat.ScopeMesh, Decision: healthDecision, Serves: watchdogsVerbs()},
|
||||
{Name: mcat.SelfCheckSeat, Scope: mcat.ScopeMesh, Decision: healthDecision, Emits: []string{"checked"}, Serves: selfCheckVerbs()},
|
||||
{Name: mcat.HealersSeat, Scope: mcat.ScopeMesh, Decision: healthDecision, Emits: []string{"healer-acted"}, Serves: healersVerbs()},
|
||||
// The control records, out of the controller (novox/hq ADR 0565 clause 1, control_records_seats.go): a person's
|
||||
// policy and the bus step at the assigner, and the hand-act log.
|
||||
{Name: mcat.WantsSeat, Scope: mcat.ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()},
|
||||
{Name: mcat.HandActsSeat, Scope: mcat.ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()},
|
||||
// Joining a machine (novox/hq ADR 0587 §1, enrolment_seat.go): held once by the module mesh-enrolment.
|
||||
{Name: mcat.EnrolmentSeat, Scope: mcat.ScopeMesh, Decision: enrolmentDecision, Emits: mcat.EnrolmentEvents, Serves: enrolmentVerbs()},
|
||||
// The artifact sweep (novox/hq ADR 0571 clause 2, artifact_sweep_seat.go).
|
||||
{Name: mcat.ArtifactSweepSeat, Scope: mcat.ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"},
|
||||
Serves: artifactSweepVerbs()},
|
||||
// Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a
|
||||
// delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight.
|
||||
// They keep their names until that migration is done deliberately, apart from the node-* pass.
|
||||
{Name: "npm-package-registry", Scope: mcat.ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"},
|
||||
{Name: "git", Scope: mcat.ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"},
|
||||
// **A build node's work, one seat per build language** (novox/hq ADR 0525 §2, §10; ADR 0536 §4; issue 1887). The
|
||||
// build module of each language claims its seat on every node it runs on; the builder and the checker put their
|
||||
// asks on the seat's accepted kinds, which the broker derives as one work queue with a shared worker per kind, and
|
||||
// each holder pulls delivery first, then check, then recheck, at most its node's build capacity at once. Its verbs
|
||||
// say what the node runs and pause or resume it there: every holder pulls from the same worker, so a node's pause is
|
||||
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
|
||||
// node-build-agent, retired with build-agent (ADR 0524 §5).
|
||||
{Name: "node-build-go", Scope: mcat.ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
{Name: "node-build-node", Scope: mcat.ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
{Name: "node-build-image", Scope: mcat.ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
|
||||
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
|
||||
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
|
||||
// requirement resolves to a holder wherever they are placed.
|
||||
{Name: "mesh-dns-resolver", Scope: mcat.ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
|
||||
Decision: "novox/hq ADR 0194, ADR 0223"},
|
||||
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
|
||||
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
|
||||
// file as the operator's, changed through these three verbs on that machine alone. The controller
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
// The machine's names are the role's files, whatever program fills it (novox/hq ADR 0287).
|
||||
Paths: []mcat.SeatPath{{Path: "/etc/hosts"}, {Path: "/etc/hostname"}},
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"cat /etc/hosts", "getent hosts"}},
|
||||
{Name: "add", Description: "Add one address and its names to the operator's lines of this machine's " +
|
||||
"/etc/hosts — a name for this machine's own programs, not the mesh's.",
|
||||
Input: schema(map[string]string{"address": "the IPv4 or IPv6 address",
|
||||
"names": "the names for it, separated by spaces"}, []string{"address", "names"}),
|
||||
Replaces: []string{"edit /etc/hosts", "HOSTALIASES"}},
|
||||
{Name: "remove", Description: "Remove one name, or every line of one address, from the operator's " +
|
||||
"lines of this machine's /etc/hosts. A line a module writes is refused, naming the module.",
|
||||
Input: schema(map[string]string{"name": "a host name, or an address to remove every line of"},
|
||||
[]string{"name"}), Replaces: []string{"sed -i /etc/hosts"}},
|
||||
}},
|
||||
// The intrusion prevention's verbs (novox/hq ADR 0179): what a person asks a machine's ban list
|
||||
// whatever keeps it — who is banned and why, ban one address, let one go. Every holder serves all
|
||||
// four; the jails themselves are composed from the modules the machine runs (to-be 31).
|
||||
{Name: "node-intrusion-prevention", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0121",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "status", Description: "Every jail on this machine with how many it is watching and " +
|
||||
"holding now, and the totals since the jail started; one jail's detail when named.",
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client status"}},
|
||||
{Name: "banned", Description: "Every address banned on this machine right now, with the jail " +
|
||||
"that holds it and when the ban ends.",
|
||||
Input: schema(map[string]string{"jail": "one jail (optional)"}, nil), Replaces: []string{"fail2ban-client banned"}},
|
||||
{Name: "ban", Description: "Ban one address in one jail now, for the jail's ban time — an " +
|
||||
"operator's act on the live ban list, which the mesh never writes itself.",
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "the jail to hold it"}, []string{"ip", "jail"}),
|
||||
Replaces: []string{"fail2ban-client set banip"}},
|
||||
{Name: "unban", Description: "Let one address go, from one jail or from every jail when none is named.",
|
||||
Input: schema(map[string]string{"ip": "the address", "jail": "one jail (optional)"}, []string{"ip"}),
|
||||
Replaces: []string{"fail2ban-client unban", "fail2ban-client set unbanip"}},
|
||||
}},
|
||||
// The packet filter's verbs (novox/hq ADR 0170): what a person asks a machine's filter whatever
|
||||
// filter answers — the rules as enforced, reload the mesh's own, remove one thing the mesh did
|
||||
// not write. Every holder serves all three; what differs by filter is the holder's own tools.
|
||||
{Name: "node-packet-filter", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0121",
|
||||
Serves: []mcat.Verb{
|
||||
{Name: "rules", Description: "The packet filter as this machine enforces it now: the nftables " +
|
||||
"ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or " +
|
||||
"chain when asked.",
|
||||
Input: schema(map[string]string{"table": "one nftables table, as `family name` (optional)",
|
||||
"chain": "one chain of that table (optional)"}, nil),
|
||||
Replaces: []string{"nft list ruleset", "iptables -L", "iptables-save"}},
|
||||
{Name: "reload", Description: "Load the mesh's own filter again from the file the mesh writes, " +
|
||||
"and answer with the mesh's table as loaded.",
|
||||
Input: schema(map[string]string{}, nil), Replaces: []string{"nft -f"}},
|
||||
{Name: "remove", Description: "Remove one rule set the mesh did not write, named exactly as the " +
|
||||
"host reports it (novox/hq ADR 0168) — `chain X (iptables-legacy)` or `table ip6 filter, chain " +
|
||||
"DOCKER-USER`. Refuses the mesh's tables, the runtime's own chains, a built-in chain and an " +
|
||||
"active found firewall's chains. An operator's act, by name, never a flush.",
|
||||
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"}),
|
||||
Replaces: []string{"nft delete", "iptables -X"}},
|
||||
}},
|
||||
// The machine's service manager (novox/hq ADR 0177). The host applies every declared unit,
|
||||
// system or user scope; the holder answers questions and operator acts about them, each verb
|
||||
// taking the unit and an optional scope. The holder runs nothing of its own: its verbs are
|
||||
// served by the node tools runtime (ADR 0175).
|
||||
{Name: mcat.ServiceManagerSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0177",
|
||||
Serves: serviceManagerVerbs()},
|
||||
// The machine's package manager (novox/hq ADR 0207). A module declaring a `package` depends on
|
||||
// it being held on its node, as one declaring a `service` depends on node-service-manager: the
|
||||
// mesh's word for "something on this machine answers for installing", where a capability only
|
||||
// says the software is there. No verbs yet — the seat says who answers, and what may be asked
|
||||
// of it is decided when someone needs to ask.
|
||||
{Name: mcat.PackageManagerSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0207"},
|
||||
// The machine's container runtime (novox/hq ADR 0166, seeded now by ADR 0207): a module
|
||||
// declaring a `container` depends on it being held on its node. Its verbs, and the host creating
|
||||
// containers through its holder, wait for ADR 0166's acceptance — seeded without them so the
|
||||
// dependency has a seat to name and the runtime's module has one to claim.
|
||||
{Name: mcat.ContainerRuntimeSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0166, ADR 0207"},
|
||||
// The operator account's environment (novox/hq ADR 0203): one module per machine writes it, and
|
||||
// every module contributes to it. No verbs — the seat says who places the environment's files,
|
||||
// and their path is its protocol: a shell sources ~/.config/mesh/environment.sh without knowing
|
||||
// which module wrote it.
|
||||
{Name: mcat.EnvironmentSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0203"},
|
||||
// The provisioner client on every machine (novox/hq ADR 0349 clause 3, design 57 P5): a service of its own on a
|
||||
// bus account of its own (mesh-controller internal/broker NodeProvisionerSeat, design 57 P3), which fills the
|
||||
// machine's templates. It serves no verb an agent calls.
|
||||
{Name: "node-provisioner", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0349"},
|
||||
// The login shell (novox/hq ADR 0204, replacing the module-declared `login-shell` of ADR 0176):
|
||||
// the mesh's, so a second shell module claims the seat rather than declaring a second one, and
|
||||
// the seat exists whether or not zsh's definition is registered. `execute` is the contract a caller
|
||||
// may call where the machine serves it (optional: ADR 0268); the holder places every module's shell
|
||||
// code in its slots.
|
||||
{Name: mcat.LoginShellSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0204",
|
||||
Serves: loginShellVerbs()},
|
||||
// A machine's power (novox/hq ADR 0211): its holder owns logind's power handling, places the
|
||||
// code modules contribute for the power moments, and publishes the machine's power states as
|
||||
// its events. Every machine has one — every machine boots and shuts down. No verbs yet.
|
||||
{Name: mcat.PowerSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0211"},
|
||||
// The machine's hotkeys (novox/hq ADR 0212): the daemon that sees the keys the window manager
|
||||
// does not — a laptop's vendor keys — run by one module per machine, which owns its
|
||||
// configuration. Every other module with keys contributes trigger lines to it.
|
||||
{Name: mcat.HotkeysSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0212",
|
||||
Receives: []mcat.Receivable{{Kind: "trigger", Comment: "#"}}},
|
||||
// The machine's message bus (novox/hq ADR 0215): its holder owns the D-Bus implementation and its
|
||||
// system service, never restarts it live, and publishes curated events about it, never its traffic.
|
||||
// It receives nothing yet: packages ship their own policies.
|
||||
{Name: mcat.MessageBusSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0215"},
|
||||
// The machine's backups (novox/hq ADR 0214, to-be 43): its holder keeps nightly restore points of
|
||||
// the data every module on the machine declares, on the machine, against mistakes rather than
|
||||
// disasters. A module contributes `backup` lines — what to run to take a consistent copy, and
|
||||
// which of its directories to keep.
|
||||
{Name: mcat.BackupSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0214",
|
||||
Serves: backupVerbs(),
|
||||
// `backup` is what to run and which paths to keep; `data` every item a module declares, with its
|
||||
// class, for the holder to measure (novox/hq ADR 0233). Both derived from modules' data sections.
|
||||
Receives: []mcat.Receivable{{Kind: mcat.BackupKindBackup, Comment: "#", Dirs: true}, {Kind: mcat.BackupKindData, Comment: "#", Dirs: true}}},
|
||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||
// model change, not a rename, so it stays until that is built.
|
||||
{Name: "the-private-network", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
|
||||
// manager and the mesh from contradicting each other — the private network's interface left
|
||||
// alone — and writes the machine's resolver file itself, because the manager is what would
|
||||
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
|
||||
// It never declares a link, an address or a wireless network, because the link is the only
|
||||
// channel a fix could arrive on. A seat
|
||||
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||
//
|
||||
// Its verbs say what the machine resolves through and over which links (novox/hq ADR 0241): the
|
||||
// resolver file as it is and who wrote it, and every link with its addresses, routes and the resolvers
|
||||
// the manager knows for it — the same for every holder, so a question about a machine's names is asked
|
||||
// the same way whatever manages its network. Optional until both holders serve them.
|
||||
//
|
||||
// It owns the resolver file, stepping back for node-resolver where that is held (novox/hq ADR 0287,
|
||||
// ADR 0247).
|
||||
{Name: "node-uplink", Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0117", Serves: uplinkVerbs(),
|
||||
Paths: []mcat.SeatPath{{Path: mcat.ResolverFile, StepsBackFor: mcat.ResolverSeat}}},
|
||||
// The machine's own resolver (novox/hq ADR 0247): held only where something requires `split-dns` — a
|
||||
// VPN client whose domains must go to its own servers while every other name goes to the mesh's.
|
||||
// Where it is held, its holder writes the resolver file and the uplink's holder steps back from it
|
||||
// (node_resolver.go). It knows nothing of any VPN: its verbs route domains to servers over a link.
|
||||
{Name: mcat.ResolverSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0247", Serves: resolverVerbs(),
|
||||
Paths: []mcat.SeatPath{{Path: mcat.ResolverFile}}},
|
||||
// A machine's shares and the shares it mounts (novox/hq ADR 0263): the holder of node-nfs-server
|
||||
// exports a machine's folders to the private network and provides each as `nfs-share`; the holder of
|
||||
// node-mounts writes a mount and an automount unit per share on a machine that asks (shares.go).
|
||||
{Name: mcat.NFSServerSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0263", Serves: nfsServerVerbs()},
|
||||
{Name: mcat.MountsSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0263", Serves: mountsVerbs()},
|
||||
// A node's files and accounts, read (novox/hq ADR 0310, reads.go): by an account of the holder's own that holds
|
||||
// no secret, on every node, so the reads ADR 0268 §5 takes over from the login shell's execute have a tool.
|
||||
{Name: mcat.ReadsSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0310", Serves: readsVerbs()},
|
||||
// A node's files, read and managed (novox/hq ADR 0394, files.go): beside node-reads until mesh-files is assigned
|
||||
// on every node, then node-reads goes (ADR 0394 §9).
|
||||
{Name: mcat.FilesSeat, Scope: mcat.ScopeNode, Decision: "novox/hq ADR 0394", Serves: filesVerbs()},
|
||||
// The node's engine's own verbs and asks (novox/hq ADR 0563 §1): held by the engine on every node, by no module.
|
||||
engineSeat(),
|
||||
},
|
||||
// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
|
||||
graphicalSessionSeats()...)
|
||||
// defaultSeats is the set the mesh ships with — the seed for the control plane's seat table and the fallback when it has none
|
||||
// (novox/hq ADR 0122). It is mesh-contracts' generated list, as mesh-sdk compiles it, taken whole: the node-engine keeps no copy of
|
||||
// a seat's verbs or events (design 26, ADR 0608 §5; issue 1966 ENG-3).
|
||||
var defaultSeats = mcat.DefaultSeats()
|
||||
|
||||
// seats is the working set the lookups read. It starts as the compiled defaults and is replaced by
|
||||
// what the control plane loaded from its store (novox/hq ADR 0122), so a change to the set is a
|
||||
@@ -467,195 +69,3 @@ func HolderAmong(provision string, providers []mcat.Provider, held []mcat.Held)
|
||||
}
|
||||
return mcat.Provider{}, false
|
||||
}
|
||||
|
||||
// serviceManagerVerbs is the contract every holder of node-service-manager serves (novox/hq ADR
|
||||
// 0177): the units on the machine in both scopes, read and acted on by name. Every verb takes an
|
||||
// optional scope — "system" when absent, "user" for the operator account's own manager — so a
|
||||
// caller asks for a user unit the way it asks for a system one; `failed` alone reads both managers
|
||||
// when none is named. `failed`, `reset-failed`, `wanted-by` and `unlink-dangling` are optional
|
||||
// (Verb.Optional) until every holder serves them (ADR 0246).
|
||||
func serviceManagerVerbs() []mcat.Verb {
|
||||
scoped := func(more map[string]string, required []string) map[string]any {
|
||||
props := map[string]string{"scope": "\"system\" (the default) or \"user\": the operator account's own manager"}
|
||||
for k, v := range more {
|
||||
props[k] = v
|
||||
}
|
||||
return schema(props, required)
|
||||
}
|
||||
unit := map[string]string{"unit": "the unit's name, as the service manager knows it"}
|
||||
return []mcat.Verb{
|
||||
{Name: "units", Description: "The units the service manager knows in a scope, each with its load, active and sub state; narrowed to a pattern when asked.",
|
||||
Input: scoped(map[string]string{"pattern": "a glob the unit's name must match (optional)"}, nil), Replaces: []string{"systemctl list-units"}},
|
||||
{Name: "status", Description: "One unit as the service manager sees it now: its states, whether it starts at boot, its main process, and whether the mesh declares it.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl status", "systemctl is-active"}},
|
||||
{Name: "start", Description: "Start one unit. For a unit the mesh declares, the answer says the host will restore what its declaration says at the next apply.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl start"}},
|
||||
{Name: "stop", Description: "Stop one unit; for a mesh-declared unit the answer says the host will restore its declared state.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl stop"}},
|
||||
{Name: "restart", Description: "Restart one unit.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl restart"}},
|
||||
{Name: "enable", Description: "Make one unit start at boot (or at the account's login, in user scope).",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl enable"}},
|
||||
{Name: "disable", Description: "Stop one unit starting at boot (or at login, in user scope).",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl disable"}},
|
||||
// **A window, not only a tail** (the operator's direction 2026-10-07): an incident is read for the
|
||||
// minutes it happened in, and with no window on the verb a person reached for a shell. Every
|
||||
// argument is the holder's to validate — passed to journalctl as one word of its own, never through
|
||||
// a shell — and what the unit printed of a secret is redacted before it is answered.
|
||||
{Name: "journal", Description: "The last lines of one unit's journal (at most 2000), in a time window and " +
|
||||
"narrowed to a priority and to lines holding a text when asked. A secret the unit printed is shown as " +
|
||||
"[redacted: <what it was>].",
|
||||
Input: scoped(map[string]string{
|
||||
"unit": unit["unit"],
|
||||
"lines": "how many lines from the end of what matches (default 100, at most 2000)",
|
||||
"since": "the window's start: an RFC 3339 time (2026-10-07T09:30:00Z) or relative to now (-30min, -2h, yesterday) (optional)",
|
||||
"until": "the window's end, in the same forms (optional; now when absent)",
|
||||
"match": "only the lines holding this text, as written — a fixed string, not a pattern (optional)",
|
||||
"priority": "only entries this severe or more: 0-7 or emerg, alert, crit, err, warning, notice, info, debug (optional)",
|
||||
}, []string{"unit"}),
|
||||
Replaces: []string{"journalctl"}},
|
||||
// What has failed, on the seat rather than as one holder's own tool: whatever holds the role answers
|
||||
// it, so a caller asks every machine the same way. **Optional while its holders catch up**: the
|
||||
// systemd module running today serves it as its own systemd_failed, and a required verb would
|
||||
// refuse it before the version serving `failed` could be delivered.
|
||||
{Name: "failed", Optional: true, Description: "Every failed unit on this machine, in the system manager and in the operator " +
|
||||
"account's; a manager that does not answer is reported with its error, never as nothing failed.",
|
||||
Input: schema(map[string]string{"scope": "\"system\" or \"user\": only that manager (both when absent)"}, nil),
|
||||
Replaces: []string{"systemctl --failed"}},
|
||||
// **Clearing a failed record, and finding what starts a unit** (novox/hq issue 332): a unit whose
|
||||
// file is gone stays failed in its manager until the record is reset, and what keeps asking for it
|
||||
// is a dependency or an enable link no verb could show. Required since every holder serves them
|
||||
// (ADR 0246 step 3).
|
||||
{Name: "reset-failed", Description: "Clear one unit's failed record in its manager — " +
|
||||
"its failed state and its start-limit count — and answer its state after. It removes no file and " +
|
||||
"starts nothing.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl reset-failed"}},
|
||||
{Name: "wanted-by", Description: "What wants, requires or triggers one unit, read-only: the " +
|
||||
"manager's reverse dependencies (WantedBy, RequiredBy, UpheldBy, BoundBy, TriggeredBy and the rest) and " +
|
||||
"every enable link naming it in the scope's configuration directories (*.wants, *.requires, *.upholds), " +
|
||||
"each with where it points and whether it dangles — the account's own directory included in user scope.",
|
||||
Input: scoped(unit, []string{"unit"}),
|
||||
Replaces: []string{"systemctl list-dependencies --reverse", "systemctl show -p WantedBy",
|
||||
"ls ~/.config/systemd/user/*.wants"}},
|
||||
// **A dangling enable link removed on the person's word** (novox/hq issue 332): `systemctl disable`
|
||||
// leaves a link whose unit file is gone, so the manager keeps asking for the unit. Required since
|
||||
// every holder serves it (ADR 0246 step 3).
|
||||
{Name: "unlink-dangling", Description: "Remove the dangling enable links named for one " +
|
||||
"unit — links in a *.wants, *.requires or *.upholds directory of the account's or the machine's own " +
|
||||
"configuration whose target does not exist, which `disable` leaves once the unit's file is gone — on " +
|
||||
"the person's word: `why` is required. Each is recorded (its path, its target, the reason) before it is " +
|
||||
"removed, and the manager reloaded; a link whose target exists, or one a package, a generator or the " +
|
||||
"runtime placed, is left and said.",
|
||||
Input: scoped(map[string]string{"unit": unit["unit"],
|
||||
"why": "the person's reason for removing the links, kept in the record"}, []string{"unit", "why"}),
|
||||
Replaces: []string{"rm ~/.config/systemd/user/*.wants/<unit>", "unlink /etc/systemd/system/*.wants/<unit>"}},
|
||||
}
|
||||
}
|
||||
|
||||
// uplinkVerbs is the contract every holder of node-uplink serves (novox/hq ADR 0241): read-only answers
|
||||
// about what the machine resolves through and over which links, whatever program manages its network.
|
||||
func uplinkVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "resolvers", Description: "The machine's resolver file as it is now: the resolvers, search " +
|
||||
"domains and options it lists, whether it is the file the mesh declares, and when it is not, who " +
|
||||
"wrote it as far as the machine shows — its header, a backup a VPN client left beside it, a link " +
|
||||
"in its place, a known writer running.",
|
||||
Input: schema(nil, nil), Optional: true},
|
||||
{Name: "links", Description: "Every network link on the machine: its state, its addresses, whether " +
|
||||
"the default route leaves through it, and the resolvers and search domains the network manager " +
|
||||
"knows for it — a VPN's tunnel included.",
|
||||
Input: schema(nil, nil), Optional: true},
|
||||
}
|
||||
}
|
||||
|
||||
// backupVerbs is the node-backup seat's protocol (novox/hq to-be 43): what is kept, take one now,
|
||||
// and restore beside the live data — never over it.
|
||||
func backupVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "backed-up", Description: "What this machine backs up: each module, what it declared, " +
|
||||
"its last good night, how many restore points are kept and the repository's size.",
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic snapshots"}},
|
||||
{Name: "now", Description: "Take a backup now, of one module or of every module on this " +
|
||||
"machine — before a migration, a retirement or anything else that could go wrong.",
|
||||
Input: schema(map[string]string{"module": "one module (optional)"}, nil), Replaces: []string{"restic backup"}},
|
||||
{Name: "restore", Description: "Restore one module's data from a restore point BESIDE the live " +
|
||||
"data, never over it: each directory as <path>.restored-<date>. Swapping it in is a " +
|
||||
"person's act. Lists the restore points when none is named.",
|
||||
Input: schema(map[string]string{
|
||||
"module": "the module",
|
||||
"snapshot": "the restore point (from `backed-up`; the newest when omitted)",
|
||||
"path": "one of the module's directories (all of them when omitted)",
|
||||
}, []string{"module"}), Replaces: []string{"restic restore"}},
|
||||
}
|
||||
}
|
||||
|
||||
// loginShellVerbs is the contract of node-login-shell (novox/hq ADR 0176, ADR 0204): one command, run
|
||||
// the way the operator's own terminal would run it, bounded below the runtime's thirty-second call limit
|
||||
// so a hung command answers rather than times the caller out.
|
||||
//
|
||||
// **`execute` is optional** (novox/hq ADR 0268). It runs any command as the operator account, which can
|
||||
// become root without a person, so a machine may withhold it: the control-node does, through the
|
||||
// holder's own `execute` setting, until a call to it needs a person's approval (hq research 039). The mark
|
||||
// is ADR 0246's: a holder that does not serve the verb on a machine still holds the seat there, and its
|
||||
// silence on the bus is not judged — a holder withholding it serves nothing on the seat.
|
||||
func loginShellVerbs() []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "execute", Optional: true, Description: "Run one command on this machine as the operator account, in a " +
|
||||
"non-interactive login shell in its home; answers with what it printed and how it exited.",
|
||||
Input: schema(map[string]string{
|
||||
"command": "the command line, as you would type it",
|
||||
"timeout_seconds": "give up after this long, at most 25 (default 20)",
|
||||
}, []string{"command"})},
|
||||
}
|
||||
}
|
||||
|
||||
// buildAskKinds are a build seat's accepted kinds, in the order a holder pulls them (ADR 0536 §4; mesh-sdk
|
||||
// delivery.AskKinds).
|
||||
func buildAskKinds() []string { return []string{"delivery", "check", "recheck"} }
|
||||
|
||||
// buildSeatVerbs are what a build module answers on its own node for its seat (ADR 0525 §10).
|
||||
func buildSeatVerbs(language string) []mcat.Verb {
|
||||
return []mcat.Verb{
|
||||
{Name: "current", Description: "The " + language + " asks this node runs now (each ask's id, kind, module and when it " +
|
||||
"started), its build capacity, and whether it is paused, taking no new ask.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "pause", Description: "Take no new " + language + " ask on this node until resumed; an ask running here " +
|
||||
"ends in its place. Kept across a restart of the build module (its pause key). A capacity of 0 is the " +
|
||||
"lasting way to keep a node from building.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "resume", Description: "Take " + language + " asks again on this node.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
}
|
||||
}
|
||||
|
||||
// vaultIntake is a two-step vault verb's arguments: what its first step names, then the call and what was sealed.
|
||||
// withHolders adds to `issued` the provision's holders from that provider (issue 749): a provider's tool may answer a
|
||||
// value for each, which the vault keeps as that holder's own and hands to it alone.
|
||||
func withHolders(input map[string]any) map[string]any {
|
||||
props, _ := input["properties"].(map[string]any)
|
||||
props["holders"] = map[string]any{"type": "array",
|
||||
"description": "the provision's holders from that provider, each {node, module}: its tool may answer a value " +
|
||||
"for each, kept as that holder's own and handed to it alone",
|
||||
"items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"node": map[string]any{"type": "string", "description": "the holder's machine"},
|
||||
"module": map[string]any{"type": "string", "description": "the consuming module"}}}}
|
||||
return input
|
||||
}
|
||||
|
||||
func vaultIntake(first map[string]string) map[string]string {
|
||||
out := map[string]string{"call": "the second step: the call the first step answered",
|
||||
"sealed": "the second step: the value sealed to the call's key — never the value"}
|
||||
for k, v := range first {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sdkSeat is the SDK's entry for a seat, taken whole, so this copy of the table cannot disagree with it.
|
||||
func sdkSeat(name string) mcat.Seat {
|
||||
seat, ok := mcat.SeatNamed(name)
|
||||
if !ok {
|
||||
panic("the SDK's seat table holds no seat " + name)
|
||||
}
|
||||
return seat
|
||||
}
|
||||
|
||||
@@ -10,7 +10,20 @@ import (
|
||||
func aBrokerClaiming(module string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: module,
|
||||
Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Serves: brokerVerbs()}}}
|
||||
}
|
||||
|
||||
// brokerVerbs are the verbs the seat mesh-broker promises its holder, as the contracts define them: a holder names every one
|
||||
// (ADR 0132), so a fixture holder does (issue 1966 ENG-3).
|
||||
func brokerVerbs() []string {
|
||||
seat, _ := catalogue.SeatNamed("mesh-broker")
|
||||
var out []string
|
||||
for _, v := range seat.Serves {
|
||||
if !v.Optional {
|
||||
out = append(out, v.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withSeat is mesh-controller's withBusSeat: a single mesh seat, mesh-broker, for the test.
|
||||
|
||||
@@ -150,6 +150,14 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
|
||||
for name, m := range base.Catalogue {
|
||||
shelf[name] = m
|
||||
}
|
||||
// A module the input's node-engine holds as refused (compose.Input.Refused: its verdict, or bytes that do not decode) is
|
||||
// held here as there, and its read says why, whatever the case's own verdicts: the check then names the cause the node
|
||||
// holds it for, never a pass the node would not compose, nor a rule about its stub (issue 774, group check 278aa5f0ccbe).
|
||||
for name, why := range base.Refused {
|
||||
if read, carried := cv.Manifests[name]; carried && read.Accepted {
|
||||
cv.Manifests[name] = ManifestRead{Refused: "its node-engine holds it: " + why}
|
||||
}
|
||||
}
|
||||
in.Catalogue = shelf
|
||||
in.Nodes = slices.Clone(in.Nodes)
|
||||
in.Holdings = slices.Clone(in.Holdings)
|
||||
@@ -298,7 +306,7 @@ const Package = "github.com/novox/mesh-host/cmd/mesh-compose"
|
||||
// Decode reads a request (sdk.Decode); it refuses one that is not JSON or of another version.
|
||||
func Decode(data []byte) (Request, error) { return sdk.Decode(data) }
|
||||
|
||||
// SentInput is the composition input a node-engine answers for the sandbox (delivery.ComposeCandidateAsk.Input), from the
|
||||
// SentInput is the composition input a node-engine answers for the sandbox (nodeengine.ComposeCandidateInput.Input), from the
|
||||
// input it composed node from: the one shape both read (issue 1126, checks of mesh-host #200). Every other node's modules
|
||||
// are left to the cases (the indexer's manifests, as the bus reader reads them too); node's own placed modules stay as the
|
||||
// bus reader took them in from their builds (their artifacts, the texts of their templates archive, what is handed to the
|
||||
|
||||
@@ -179,6 +179,47 @@ func TestARequestItCannotRunIsRefused(t *testing.T) {
|
||||
t.Errorf("a module the node-engine holds too: %+v", m)
|
||||
}
|
||||
})
|
||||
// Issue 774 (the group check of feat/l3-final 278aa5f0ccbe): a tool runner its node-engine holds by its manifest's
|
||||
// verdict is sent as its stub, and the input says it holds it: the sandbox composes the node with it held, never refuses
|
||||
// it for its stub's missing bundle. An input from an engine before the mark is read by Refused alone, the same way.
|
||||
t.Run("a refused tool runner the engine sends as its stub", func(t *testing.T) {
|
||||
raw := `{"module": "runner", "version": "1", "tool-runner": true, ` +
|
||||
`"build": {"artifacts": [{"name": "runtime", "kind": "bundle", "language": "go", "from": "cmd/runner", "binary": "runner"}]}}`
|
||||
man, err := catalogue.Decode([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{"runner": compose.Stub(man)},
|
||||
Refused: map[string]string{"runner": "the indexer's judge refused the manifest of runner"},
|
||||
Nodes: []compose.Node{{Name: "a", Assigned: []string{"runner"}}}}
|
||||
wire, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back compose.Input
|
||||
if err := json.Unmarshal(wire, &back); err != nil || !back.Catalogue["runner"].Stubbed || !strings.Contains(string(wire), `"Held"`) {
|
||||
t.Fatalf("the input's held mark across the wire: %s, %v", wire, err)
|
||||
}
|
||||
older := strings.Replace(string(wire), `"Held":{"Catalogue":["runner"]}`, `"Unsaid":null`, 1)
|
||||
for name, sent := range map[string]string{"with the mark": string(wire), "from an engine before it": older} {
|
||||
var r Request
|
||||
r = Request{V: Version, Machines: []string{"a"}, Depth: DepthCompose,
|
||||
Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"runner": {Raw: json.RawMessage(raw)}}}}}
|
||||
if err := json.Unmarshal([]byte(sent), &r.Input); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := Run(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := v.Cases[0].Machines["a"]; !m.Composes || strings.Contains(m.Reason, "bundle") {
|
||||
t.Errorf("%s: the node with its tool runner held: %+v", name, m)
|
||||
}
|
||||
if read := v.Cases[0].Manifests["runner"]; read.Accepted || !strings.Contains(read.Refused, "the indexer's judge refused the manifest of runner") {
|
||||
t.Errorf("%s: the runner's read names not why its node-engine holds it: %+v", name, read)
|
||||
}
|
||||
}
|
||||
})
|
||||
v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}})
|
||||
if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" {
|
||||
t.Fatalf("a request with no machines: %+v, %v", v, err)
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
|
||||
@@ -78,7 +78,7 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant
|
||||
warrant func(asked oc.Ask, w oc.Warrant) oc.Warrant
|
||||
diskAfter string
|
||||
twice bool
|
||||
done int
|
||||
@@ -87,19 +87,19 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
{name: "no warrant", done: 0, state: HandAsked},
|
||||
{name: "the operator's yes", warrant: yes, done: 1, state: HandDone},
|
||||
{name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone},
|
||||
{name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
{name: "a warrant for another digest", warrant: func(a oc.Ask, w oc.Warrant) oc.Warrant {
|
||||
w = yes(a, w)
|
||||
w.AskDigest = "sha256:another"
|
||||
return w
|
||||
}, done: 0, state: HandAsked},
|
||||
{name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
{name: "a no", warrant: func(a oc.Ask, w oc.Warrant) oc.Warrant {
|
||||
w = yes(a, w)
|
||||
w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge
|
||||
w.Option, w.Label, w.Level = "no", "No, leave it", occ.Acknowledge
|
||||
return w
|
||||
}, done: 0, state: HandRefused},
|
||||
{name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused},
|
||||
{name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At}
|
||||
{name: "expired", warrant: func(a oc.Ask, w oc.Warrant) oc.Warrant {
|
||||
return oc.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: occ.OutcomeExpired, At: w.At}
|
||||
}, done: 0, state: HandRefused},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
@@ -130,7 +130,7 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
disk = row.diskAfter
|
||||
}
|
||||
if row.warrant != nil {
|
||||
w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()})
|
||||
w := row.warrant((*asked)[0], oc.Warrant{Asker: occ.EngineAsker("n1"), At: occ.FormatTime(time.Now())})
|
||||
h.Heard(w)
|
||||
if row.twice {
|
||||
h.Heard(w)
|
||||
@@ -153,25 +153,25 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
}
|
||||
|
||||
// yes is the operator's yes to an ask, as the router writes it.
|
||||
func yes(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve
|
||||
w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}
|
||||
w.AskDigest, w.Proofs = a.Digest(), []string{"P1"}
|
||||
func yes(a oc.Ask, w oc.Warrant) oc.Warrant {
|
||||
w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, occ.OutcomeChosen, "yes", "Yes", occ.Approve
|
||||
w.By = &oc.Person{Who: occ.Operator, Kind: "telegram", Verified: "user id verified"}
|
||||
w.AskDigest, w.Proofs = occ.Digest(a), []string{"P1"}
|
||||
return w
|
||||
}
|
||||
|
||||
// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a
|
||||
// hand-over that counts what it performed.
|
||||
func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) {
|
||||
func anEngine(t *testing.T) (*HandActs, *[]oc.Ask, *int) {
|
||||
t.Helper()
|
||||
var asked []asks.Ask
|
||||
var asked []oc.Ask
|
||||
done := 0
|
||||
state := &MemState{}
|
||||
h := &HandActs{Node: "n1", State: state,
|
||||
HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }}
|
||||
h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||
h.Asker = &occ.Client{Module: occ.EngineAsker("n1"), State: state,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) {
|
||||
if a, ok := body.(oc.Ask); ok && subject == oc.AcceptSubjectAsk(occ.EngineAsker("n1")) {
|
||||
asked = append(asked, a)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
@@ -192,7 +192,7 @@ func (l *natsLink) SayEngine(event string, body any) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR
|
||||
// The engine asks the operator under its own asker name (occ.EngineAsker) on the router's seat (ADR 0259 §3, as ADR
|
||||
// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and
|
||||
// its asks' state read from the router's records under its own name.
|
||||
|
||||
@@ -204,7 +204,7 @@ const RouterRecords = "messenger_asks"
|
||||
type OperatorAsks interface {
|
||||
PublishAsk(subject string, body any, id string) error
|
||||
RouterRecord(key string) (*stdio.StateEntry, error)
|
||||
HearWarrants(asker string, heard func(asks.Warrant)) error
|
||||
HearWarrants(asker string, heard func(oc.Warrant)) error
|
||||
}
|
||||
|
||||
// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window.
|
||||
@@ -236,9 +236,9 @@ func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||
}
|
||||
|
||||
// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives.
|
||||
func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error {
|
||||
sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) {
|
||||
var w asks.Warrant
|
||||
func (l *natsLink) HearWarrants(asker string, heard func(oc.Warrant)) error {
|
||||
sub, err := l.conn.Subscribe(oc.SubjectDecided(asker), func(msg *nats.Msg) {
|
||||
var w oc.Warrant
|
||||
if json.Unmarshal(msg.Data, &w) == nil {
|
||||
heard(w)
|
||||
}
|
||||
|
||||
+51
-37
@@ -13,18 +13,20 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The node seat node-engine's acts (novox/hq ADR 0563).
|
||||
//
|
||||
// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing.
|
||||
// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator
|
||||
// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every
|
||||
// under its own asker name (occ.EngineAsker) at the level approve, binding the digest of the verb, the node, every
|
||||
// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own
|
||||
// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now;
|
||||
// state (occ.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now;
|
||||
// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and
|
||||
// changes nothing. One ask is open per path or access.
|
||||
//
|
||||
@@ -32,28 +34,30 @@ import (
|
||||
// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own
|
||||
// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts.
|
||||
|
||||
// The acts of the seat, as its verbs name them.
|
||||
// The acts of the seat, as its verbs name them: the seat's contract's names (mesh-contracts, seats/node-engine).
|
||||
const (
|
||||
ActHandOver = "hand-over"
|
||||
ActAccess = "access-act"
|
||||
ActJudgeDirectories = "judge-directories"
|
||||
ActSetuidSearch = "setuid-search"
|
||||
ActHandOver = nodeengine.VerbHandOver
|
||||
ActAccess = nodeengine.VerbAccessAct
|
||||
ActJudgeDirectories = nodeengine.VerbJudgeDirectories
|
||||
ActSetuidSearch = nodeengine.VerbSetuidSearch
|
||||
// ActComposition reads a module's composition beside what runs of it; read-only, no warrant (issue 774).
|
||||
ActComposition = "composition"
|
||||
ActComposition = nodeengine.VerbComposition
|
||||
// ActCompose composes the node now, the recovery of a failed composition (issue 774).
|
||||
ActCompose = "compose"
|
||||
ActCompose = nodeengine.VerbCompose
|
||||
// ActPackages reads the packages this node's compositions declare, and with write writes its key again (novox/hq
|
||||
// ADR 0565 §6, §15).
|
||||
ActPackages = "packages"
|
||||
ActPackages = nodeengine.VerbPackages
|
||||
// ActComposeCandidate composes a check's candidate on this node in shadow, read-only (novox/hq ADR 0593 §7).
|
||||
ActComposeCandidate = "compose-candidate"
|
||||
ActComposeCandidate = nodeengine.VerbComposeCandidate
|
||||
// ActManifestReads reads the reader capabilities this engine states (novox/hq ADR 0618 §3).
|
||||
ActManifestReads = nodeengine.VerbManifestReads
|
||||
)
|
||||
|
||||
// The states of an act.
|
||||
// The states of an act: asked, done and refused are also the seat's events.
|
||||
const (
|
||||
HandAsked = "asked"
|
||||
HandDone = "done"
|
||||
HandRefused = "refused"
|
||||
HandAsked = nodeengine.EventAsked
|
||||
HandDone = nodeengine.EventDone
|
||||
HandRefused = nodeengine.EventRefused
|
||||
HandAnswered = "answered"
|
||||
)
|
||||
|
||||
@@ -122,8 +126,8 @@ type HandActs struct {
|
||||
Node string
|
||||
// State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act.
|
||||
State ActStore
|
||||
// Asker asks the operator as asks.EngineAsker(Node), with State as its store.
|
||||
Asker *asker.Client
|
||||
// Asker asks the operator as occ.EngineAsker(Node), with State as its store.
|
||||
Asker *occ.Client
|
||||
// JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant.
|
||||
JudgeHandOver func(path string) (Judged, error)
|
||||
HandOver HandOverFunc
|
||||
@@ -149,6 +153,9 @@ type HandActs struct {
|
||||
// module, applying and writing nothing; with input, also the composition input it composed from (the verb
|
||||
// compose-candidate, ADR 0593 §7).
|
||||
ComposeCandidate func(candidate string, input bool) (any, error)
|
||||
// ManifestReads is what this engine states it reads of a manifest and when its record last carried them (the verb
|
||||
// manifest-reads, novox/hq ADR 0618 §3).
|
||||
ManifestReads func() ([]string, time.Time)
|
||||
// Say says one of the seat's events, asked, done or refused; nil says nothing.
|
||||
Say func(event string, body any)
|
||||
Now func() time.Time
|
||||
@@ -230,6 +237,13 @@ func (h *HandActs) Call(c EngineCall) (any, error) {
|
||||
input = b
|
||||
}
|
||||
return h.ComposeCandidate(arg("candidate"), input)
|
||||
case ActManifestReads:
|
||||
if h.ManifestReads == nil {
|
||||
return nil, errors.New("this engine states no manifest reads yet: its node agent has not started")
|
||||
}
|
||||
reads, at := h.ManifestReads()
|
||||
return nodeengine.ManifestReadsAnswer{V: delivery.Version, Node: h.Node, ManifestReads: reads,
|
||||
StatedAt: at.UTC().Format(time.RFC3339Nano)}, nil
|
||||
case "acts":
|
||||
if id := arg("id"); id != "" {
|
||||
return h.Act(id)
|
||||
@@ -515,8 +529,8 @@ func (h *HandActs) id(act string) string {
|
||||
}
|
||||
|
||||
// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3).
|
||||
func (h *HandActs) bound(a HandAct) asks.Act {
|
||||
act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets}
|
||||
func (h *HandActs) bound(a HandAct) occ.Act {
|
||||
act := occ.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets}
|
||||
if a.Path != "" {
|
||||
act["path"] = a.Path
|
||||
}
|
||||
@@ -577,7 +591,7 @@ func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error
|
||||
defer h.mu.Unlock()
|
||||
now := h.now().UTC()
|
||||
a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now,
|
||||
State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto}
|
||||
State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(occ.ApproveLasts - time.Minute), Auto: auto}
|
||||
a.Ask = a.ID
|
||||
// One ask open per path or access: its open key made by a create, which a second ask cannot make.
|
||||
if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) {
|
||||
@@ -598,7 +612,7 @@ func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error
|
||||
_ = h.State.Delete(openKey(act, j.Subject))
|
||||
return nil, err
|
||||
}
|
||||
bind, err := asks.ActDigest(h.bound(a))
|
||||
bind, err := occ.ActDigest(h.bound(a))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -609,16 +623,16 @@ func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error
|
||||
"The library's group, group write and setgid are set on everything under its root.",
|
||||
"The library waits and nothing under its root changes."
|
||||
}
|
||||
if len([]rune(headline)) > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
if len([]rune(headline)) > occ.HeadlineLength {
|
||||
headline = string([]rune(headline)[:occ.HeadlineLength])
|
||||
}
|
||||
asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator,
|
||||
asked := oc.Ask{ID: a.Ask, Headline: headline, Who: occ.Operator,
|
||||
Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||
Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||
Details: "Asked by " + caller + " on " + h.Node + orWhy(why),
|
||||
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
|
||||
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
|
||||
Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act}
|
||||
Options: []oc.Option{{ID: "yes", Label: "Yes", Does: does, Level: occ.Approve, Binds: bind},
|
||||
{ID: "no", Label: "No, leave it", Does: stays, Level: occ.Acknowledge}},
|
||||
Expires: occ.FormatTime(a.Expires), OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act}
|
||||
if err := h.Asker.Ask(asked); err != nil {
|
||||
_, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() })
|
||||
return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err)
|
||||
@@ -639,10 +653,10 @@ func orWhy(why string) string {
|
||||
return ": " + why
|
||||
}
|
||||
|
||||
// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the
|
||||
// Warranted is the act on the operator's answer, handed once by occ.Client.Take after it held the warrant to the
|
||||
// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now,
|
||||
// and only then performed and recorded done with the warrant.
|
||||
func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
func (h *HandActs) Warranted(o oc.Option, w oc.Warrant) error {
|
||||
a, _, err := h.readOne(actKey(w.Ask))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -674,7 +688,7 @@ func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
}
|
||||
now := *a
|
||||
now.Sets, now.Path = j.Sets, j.Path
|
||||
if err := o.Performs(h.bound(now)); err != nil {
|
||||
if err := occ.Performs(o, h.bound(now)); err != nil {
|
||||
return refuse("what the yes bound is not what would be done now: " + err.Error())
|
||||
}
|
||||
if j.Disk != a.Disk {
|
||||
@@ -699,7 +713,7 @@ func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func warrantOn(a *HandAct, w asks.Warrant) {
|
||||
func warrantOn(a *HandAct, w oc.Warrant) {
|
||||
a.Proofs = w.Proofs
|
||||
if w.By != nil {
|
||||
a.By, a.Via = w.By.Who, w.By.Kind
|
||||
@@ -709,22 +723,22 @@ func warrantOn(a *HandAct, w asks.Warrant) {
|
||||
}
|
||||
}
|
||||
|
||||
func operatorOf(w asks.Warrant) string {
|
||||
func operatorOf(w oc.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return "the " + w.By.Who + " via " + w.By.Kind
|
||||
}
|
||||
|
||||
// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with
|
||||
// Heard is a warrant heard on the engine's decided subject: taken once (occ.Client.Take), and an ask that ended with
|
||||
// no choice recorded refused.
|
||||
func (h *HandActs) Heard(w asks.Warrant) {
|
||||
func (h *HandActs) Heard(w oc.Warrant) {
|
||||
if h.Asker == nil {
|
||||
return
|
||||
}
|
||||
// A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no
|
||||
// answer to the ask, which stays open for the operator's.
|
||||
if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded {
|
||||
if handled, _ := h.Asker.Take(w, h.Warranted); handled == occ.Recorded {
|
||||
why := "the ask ended " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
why += ": " + w.Words
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
@@ -191,6 +192,23 @@ type handingLink struct {
|
||||
given chan HandOverFunc
|
||||
}
|
||||
|
||||
// readsLink is a quiet link that serves the node seat and keeps the manifest-reads-changed events said on it.
|
||||
type readsLink struct {
|
||||
*quietLink
|
||||
said chan nodeengine.ManifestReadsChangedEvent
|
||||
}
|
||||
|
||||
func (l readsLink) ServeEngineSeat(EngineFunc) {}
|
||||
func (l readsLink) SayEngine(event string, body any) error {
|
||||
// As the bus carries it: the body written and read back as the seat's contract.
|
||||
raw, err := json.Marshal(body)
|
||||
var e nodeengine.ManifestReadsChangedEvent
|
||||
if err == nil && event == nodeengine.EventManifestReadsChanged && json.Unmarshal(raw, &e) == nil {
|
||||
l.said <- e
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
||||
|
||||
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
||||
@@ -213,6 +231,30 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||
}
|
||||
stop()
|
||||
<-served
|
||||
// ADR 0618 §4: an engine that holds the seat says manifest-reads-changed once for each link it serves, "started" on
|
||||
// the first and "returned" on every one after it, and never once per pass.
|
||||
said := make(chan nodeengine.ManifestReadsChangedEvent, 4)
|
||||
for _, why := range []string{ReadsStarted, ReadsReturned} {
|
||||
el := readsLink{quietLink: newQuietLink(), said: said}
|
||||
eq := q
|
||||
eq.Acts = &HandActs{Node: "n1", State: &MemState{}}
|
||||
ectx, estop := context.WithCancel(context.Background())
|
||||
edone := make(chan error, 1)
|
||||
go func() { edone <- serve(ectx, el, m, eq, nil, time.Second) }()
|
||||
select {
|
||||
case got := <-said:
|
||||
if at, err := time.Parse(time.RFC3339Nano, got.At); got.Why != why || got.Node != "n1" || got.V != 1 || err != nil || at.IsZero() {
|
||||
t.Errorf("the link said %+v, want %s", got, why)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the link %s said nothing", why)
|
||||
}
|
||||
estop()
|
||||
<-edone
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Errorf("more than once per link: %d more said", len(said))
|
||||
}
|
||||
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
|
||||
// the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope.
|
||||
t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) {
|
||||
@@ -223,6 +265,10 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||
return map[string]any{"module": module, "digest": digest, "diff": diff}, nil
|
||||
},
|
||||
Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil },
|
||||
// ADR 0618 §3: what this engine states it reads, and when it last stated it.
|
||||
ManifestReads: func() ([]string, time.Time) {
|
||||
return []string{"form:a", "seat:b"}, time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC)
|
||||
},
|
||||
// ADR 0565 §6, §15: the node's key of the engine's state packages.
|
||||
Packages: func(write bool) (any, error) {
|
||||
key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`)
|
||||
@@ -245,11 +291,14 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||
// The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named.
|
||||
{"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`},
|
||||
{"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`},
|
||||
// ENG-9's pull waits are carried in the failed have, read through node_haves: the seat serves no verb of them.
|
||||
{"pull-waits", EngineToolSubject("pull-waits", "n1"), `"error":"the node seat node-engine serves no verb \"pull-waits\""`},
|
||||
// The read verb packages (ADR 0565 §6) names its source; with write, what it wrote, and a key the engine has not
|
||||
// written is said and not an error.
|
||||
{"packages", EngineToolSubject("packages", "n1"), `"source":"mesh-host.packages","key":{"node":"n1"`},
|
||||
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`},
|
||||
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`},
|
||||
{"manifest-reads", EngineToolSubject("manifest-reads", "n1"), `"manifest-reads":["form:a","seat:b"],"node":"n1","stated-at":"2026-11-04T10:00:00Z","v":1`},
|
||||
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
|
||||
} {
|
||||
subject, body, ok := strings.Cut(row.subject, " ")
|
||||
|
||||
@@ -296,7 +296,9 @@ func (q *Session) noteApply(r Report) bool {
|
||||
func (q *Session) nodeFactsWithRecord(r Report, before *delivery.HaveNode) *delivery.HaveNode {
|
||||
n := nodeFacts(r)
|
||||
if q.Keys != nil {
|
||||
n.Record = recordOf(q.Membership.Node, r, q.Keys())
|
||||
keys := q.Keys()
|
||||
keys.Stated = q.runnerStated
|
||||
n.Record = recordOf(q.Membership.Node, r, keys)
|
||||
}
|
||||
if before != nil && before.Record != nil {
|
||||
switch {
|
||||
@@ -385,6 +387,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
h.Phase = delivery.PhaseFailed
|
||||
}
|
||||
h.SealedAt = f.Sealed
|
||||
if h.Phase == delivery.PhaseFailed {
|
||||
h.PullWait = f.PullWait
|
||||
}
|
||||
h.Health = s.healthOf(module, f)
|
||||
if h.Detail == "" {
|
||||
h.Detail = s.wordsOf(module, h.Health)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -31,6 +32,15 @@ type haveLink struct {
|
||||
bindFails, bindStuck bool
|
||||
// bindNoResponders fails that many binds with the bus's no responders, as while it reloads its accounts.
|
||||
bindNoResponders int
|
||||
// stated is what the tool runner's PING says of itself (Stater); statedErr is the runner not answering.
|
||||
stated map[string]string
|
||||
statedErr error
|
||||
}
|
||||
|
||||
func (l *haveLink) Stated(context.Context, string, string) (map[string]string, error) {
|
||||
l.hmu.Lock()
|
||||
defer l.hmu.Unlock()
|
||||
return l.stated, l.statedErr
|
||||
}
|
||||
|
||||
type haveEntry struct {
|
||||
@@ -259,6 +269,7 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
|
||||
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
|
||||
q.Keys = func() RecordKeys { return keys }
|
||||
l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls"} // the tool runner's calls (ENG-2)
|
||||
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
||||
"machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}}
|
||||
@@ -269,8 +280,9 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
||||
}
|
||||
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
||||
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
||||
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract,
|
||||
ManifestReads: catalogue.ManifestReads()} // what it reads of a manifest: its contract's (ADR 0614 §4)
|
||||
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
||||
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
||||
}
|
||||
@@ -279,6 +291,22 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
if l.kv["n1.node"].rev != at {
|
||||
t.Errorf("a report changing nothing wrote the node key again")
|
||||
}
|
||||
// A runner that does not answer states what it stated before; one answering without any states none (an older runner).
|
||||
l.statedErr = fmt.Errorf("%w: nothing answered", ErrNoAnswer)
|
||||
l.stated = nil
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if l.kv["n1.node"].rev != at {
|
||||
t.Errorf("a runner that did not answer changed the record")
|
||||
}
|
||||
l.statedErr, l.stated = nil, map[string]string{}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd"}) {
|
||||
t.Errorf("a runner stating no calls: revision %d (was %d), capabilities %v", l.kv["n1.node"].rev, at, got.Record.Capabilities)
|
||||
}
|
||||
at++
|
||||
l.stated = map[string]string{CapabilitiesKey: "ledger-calls,bus-state-calls"}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
at++
|
||||
keys.Certificate = "cert-2"
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || got.Record.Certificate != "cert-2" {
|
||||
@@ -553,6 +581,14 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
|
||||
t.Fatalf("an apply lost the user list: %+v", got)
|
||||
}
|
||||
|
||||
// A failed apply waiting on a store's holder carries the pull it waits on in its have (ENG-9, read by node_haves).
|
||||
wait := &delivery.PullWait{Image: "reg/x@sha256:1", Seat: "container-image-registry", Holder: "distribution",
|
||||
Began: time.Date(2026, 11, 4, 14, 0, 0, 0, time.UTC), Retried: "distribution settled"}
|
||||
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Failed: true, Detail: "the pull failed", PullWait: wait})
|
||||
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseFailed || got.PullWait == nil || got.PullWait.Holder != "distribution" {
|
||||
t.Fatalf("the failed have does not say the pull it waits on: %+v", got)
|
||||
}
|
||||
|
||||
// A module held for a failed backup says so and what it keeps running; removed is written once, over its key.
|
||||
q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseHeld), Detail: "the repository is locked"})
|
||||
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseHeld || got.Detail == "" {
|
||||
|
||||
@@ -645,4 +645,6 @@ type ModuleFact struct {
|
||||
Kept []string
|
||||
// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
|
||||
Sealed delivery.Seals
|
||||
// PullWait is, on a failed fact, the pull from a store its apply waits on (delivery.Have.PullWait, issue 1966 ENG-9).
|
||||
PullWait *delivery.PullWait
|
||||
}
|
||||
|
||||
+21
-1
@@ -2,7 +2,10 @@ package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -25,8 +28,19 @@ type RecordKeys struct {
|
||||
Authority string
|
||||
// Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2).
|
||||
Trusts []string
|
||||
// Stated is the capabilities its tool runner states of the calls it answers (CapabilitiesKey), which the record says
|
||||
// beside the machine's own: a module whose manifest asks for one is held off a machine whose runner does not state it.
|
||||
Stated []string
|
||||
}
|
||||
|
||||
// manifestReads is what this engine reads of a manifest (novox/hq ADR 0614 §4): the reader capabilities of the mesh-sdk it
|
||||
// is built with, stated in its record so the assigner moves a want only to an engine that reads its manifest. Computed
|
||||
// once: it is the build's, and never changes while it runs.
|
||||
var manifestReads = sync.OnceValue(catalogue.ManifestReads)
|
||||
|
||||
// readsStatedAt is when this engine computed what it reads: its start (the verb manifest-reads, ADR 0618 §3).
|
||||
var readsStatedAt = time.Now().UTC()
|
||||
|
||||
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
||||
// says nothing of the machine, such as a rekey), so the record last written stands.
|
||||
func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
||||
@@ -47,12 +61,18 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
||||
return nil
|
||||
}
|
||||
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts}
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts,
|
||||
ManifestReads: manifestReads()}
|
||||
for _, c := range p.Capabilities {
|
||||
if c.Present && c.Name != "" {
|
||||
rec.Capabilities = append(rec.Capabilities, c.Name)
|
||||
}
|
||||
}
|
||||
for _, c := range keys.Stated {
|
||||
if c != "" && !slices.Contains(rec.Capabilities, c) {
|
||||
rec.Capabilities = append(rec.Capabilities, c)
|
||||
}
|
||||
}
|
||||
sort.Strings(rec.Capabilities)
|
||||
if p.Machine != nil {
|
||||
rec.NetworksSaid = true
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
meshasks "git.novox.be/novox/mesh-sdk/go/asks"
|
||||
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
||||
)
|
||||
|
||||
// Membership is what a node needs to reach its mesh again, held by the caller.
|
||||
@@ -218,9 +218,11 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
||||
if queue.Acts != nil {
|
||||
if s, ok := link.(EngineSeats); ok {
|
||||
s.ServeEngineSeat(queue.Acts.Call)
|
||||
// And what it reads of a manifest, said once for this link (novox/hq ADR 0618 §4).
|
||||
queue.SayReads()
|
||||
}
|
||||
if a, ok := link.(OperatorAsks); ok {
|
||||
if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil {
|
||||
if err := a.HearWarrants(occ.EngineAsker(m.Node), queue.Acts.Heard); err != nil {
|
||||
say("cannot hear the operator's answers to this engine's asks: " + err.Error())
|
||||
}
|
||||
go queue.Acts.Reconcile()
|
||||
|
||||
@@ -5,11 +5,13 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
@@ -79,9 +81,19 @@ type Session struct {
|
||||
haveAgainPending bool
|
||||
haveAgainTries int
|
||||
|
||||
// runnerStated is the capabilities this machine's tool runner last stated of the calls it answers (askRunner), said in the
|
||||
// node's record. Held under saying.
|
||||
runnerStated []string
|
||||
|
||||
// saying is held while a facts event is noted, so the haves are written in the order the facts are made.
|
||||
saying sync.Mutex
|
||||
|
||||
// reads says whether the engine has said its manifest reads on a link before (novox/hq ADR 0618 §4).
|
||||
reads struct {
|
||||
sync.Mutex
|
||||
linked bool
|
||||
}
|
||||
|
||||
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
||||
haves haveState
|
||||
// healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone
|
||||
@@ -161,6 +173,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
|
||||
return false
|
||||
}
|
||||
r.Node = q.Membership.Node
|
||||
q.askRunner(ctx, bus)
|
||||
if q.noteApply(r) {
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
@@ -377,6 +390,33 @@ func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||
return asking.RouterRecord(key)
|
||||
}
|
||||
|
||||
// StatedReads is what this engine states it reads of a manifest and when it computed them, which is its start (the
|
||||
// node seat's verb manifest-reads, novox/hq ADR 0618 §3): the record is written only when it differs, so its put says
|
||||
// nothing of when an unchanged statement was made.
|
||||
func (q *Session) StatedReads() ([]string, time.Time) { return manifestReads(), readsStatedAt }
|
||||
|
||||
// The reasons the engine says its reads, in the event's why (novox/hq ADR 0618 §4): it started, or its node returned
|
||||
// after being away (a link after the first).
|
||||
const (
|
||||
ReadsStarted = "started"
|
||||
ReadsReturned = "returned"
|
||||
)
|
||||
|
||||
// SayReads says the node seat's event manifest-reads-changed once for the link just opened (novox/hq ADR 0618 §4):
|
||||
// why "started" on the engine's first link and "returned" on every link after it, so a reader asks the verb again once
|
||||
// per connection and never once per pass.
|
||||
func (q *Session) SayReads() {
|
||||
q.reads.Lock()
|
||||
why := ReadsStarted
|
||||
if q.reads.linked {
|
||||
why = ReadsReturned
|
||||
}
|
||||
q.reads.linked = true
|
||||
q.reads.Unlock()
|
||||
q.SayEngine(nodeengine.EventManifestReadsChanged, nodeengine.ManifestReadsChangedEvent{V: delivery.Version,
|
||||
Node: q.Membership.Node, ManifestReads: manifestReads(), At: time.Now().UTC().Format(time.RFC3339Nano), Why: why})
|
||||
}
|
||||
|
||||
// SayEngine says one of the node seat's events over the link open now; with none open it is not said.
|
||||
func (q *Session) SayEngine(event string, body any) {
|
||||
q.init()
|
||||
@@ -404,3 +444,30 @@ func (q *Session) operatorAsks() (OperatorAsks, error) {
|
||||
}
|
||||
return asking, nil
|
||||
}
|
||||
|
||||
// runnerAsk bounds the question to the tool runner.
|
||||
const runnerAsk = 3 * time.Second
|
||||
|
||||
// askRunner reads what this machine's tool runner states of the calls it answers, asked of its own instance with the PING
|
||||
// the witness asks (ENG-2, issue 1966), at each report: a runner replaced by a build that states more, or less, is said in
|
||||
// the record from the next report. A runner that does not answer states what it stated before, since its absence says
|
||||
// nothing of its calls; one that answers without stating any states none, as an older runner does. Called under saying.
|
||||
func (q *Session) askRunner(ctx context.Context, bus Bus) {
|
||||
stater, ok := bus.(Stater)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, runnerAsk)
|
||||
defer cancel()
|
||||
md, err := stater.Stated(asking, ToolRunnerService, q.Membership.Node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var stated []string
|
||||
for _, c := range strings.Split(md[CapabilitiesKey], ",") {
|
||||
if c = strings.TrimSpace(c); c != "" {
|
||||
stated = append(stated, c)
|
||||
}
|
||||
}
|
||||
q.runnerStated = stated
|
||||
}
|
||||
|
||||
@@ -47,27 +47,50 @@ func (q *Session) Asker() Asker {
|
||||
}
|
||||
|
||||
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
|
||||
_, err := l.stated(ctx, service, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ToolRunnerService is the services-protocol name the tool runner answers discovery under, and CapabilitiesKey the metadata
|
||||
// key of its PING it states the capabilities of the calls it answers under, comma-separated (mesh-tools announce).
|
||||
const (
|
||||
ToolRunnerService = "node-tools"
|
||||
CapabilitiesKey = "capabilities"
|
||||
)
|
||||
|
||||
// Stater is a link that can ask what a service's instance states of itself in its PING: its metadata.
|
||||
type Stater interface {
|
||||
Stated(ctx context.Context, service, id string) (map[string]string, error)
|
||||
}
|
||||
|
||||
// Stated is what the tool runner of this machine states in its PING, asked of its own instance as a witness asks it.
|
||||
func (l *natsLink) Stated(ctx context.Context, service, id string) (map[string]string, error) {
|
||||
return l.stated(ctx, service, id)
|
||||
}
|
||||
|
||||
func (l *natsLink) stated(ctx context.Context, service, id string) (map[string]string, error) {
|
||||
subject := PingSubject(service, id)
|
||||
before := l.conn.LastError()
|
||||
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
// The bus delivered the question and nothing subscribes to it: the runtime is not on the bus.
|
||||
return fmt.Errorf("%w: no %s on this machine is on the bus", ErrNoAnswer, service)
|
||||
return nil, fmt.Errorf("%w: no %s on this machine is on the bus", ErrNoAnswer, service)
|
||||
case err != nil:
|
||||
if refused := l.refusal(subject, before); refused != "" {
|
||||
return fmt.Errorf("%w: asking %s: %v%s", ErrCannotAsk, subject, err, refused)
|
||||
return nil, fmt.Errorf("%w: asking %s: %v%s", ErrCannotAsk, subject, err, refused)
|
||||
}
|
||||
return fmt.Errorf("%w: %s did not answer: %v", ErrNoAnswer, subject, err)
|
||||
return nil, fmt.Errorf("%w: %s did not answer: %v", ErrNoAnswer, subject, err)
|
||||
}
|
||||
var ping struct {
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &ping); err != nil || ping.Name != service || ping.ID != id {
|
||||
return fmt.Errorf("%w: what answered %s is not %s %s", ErrNoAnswer, subject, service, id)
|
||||
return nil, fmt.Errorf("%w: what answered %s is not %s %s", ErrNoAnswer, subject, service, id)
|
||||
}
|
||||
return nil
|
||||
return ping.Metadata, nil
|
||||
}
|
||||
|
||||
// refusal is the bus's word on this connection when it refused a publish to subject while this question
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -11,6 +12,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
@@ -32,11 +35,11 @@ import (
|
||||
var errNotReady = errors.New("this node's shadow has not read every input yet")
|
||||
|
||||
// candidateWithin is the node's own bound on an answer to compose-candidate: shorter than the checker's
|
||||
// (delivery.ComposeAnswerBound), because the bus lets the engine answer an ask only within a minute of it (the grants
|
||||
// (nec.AnswerBound), because the bus lets the engine answer an ask only within a minute of it (the grants
|
||||
// part's allow_responses ttl, mesh-grants ResponseTTL): an answer at the checker's 60 s bound was refused by the bus
|
||||
// (issue 1126, 2026-11-03 01:17:04 on shanks, "Permissions Violation for Publish to _INBOX..."), so the checker heard
|
||||
// nothing at all. A node not done by then answers unready, naming what it waits on.
|
||||
var candidateWithin = delivery.ComposeAnswerBound - 10*time.Second
|
||||
var candidateWithin = nec.AnswerBound - 10*time.Second
|
||||
|
||||
// maxSentInput is the most composition input an answer carries: the bus carries 1 MiB in one message (its max_payload),
|
||||
// and the answer's own fields and envelope come beside it.
|
||||
@@ -91,7 +94,7 @@ func (r *candidateRun) said(now time.Time) (waits string, on string) {
|
||||
// ComposeCandidate is the node's answer to ask, within candidateWithin (or ctx's deadline, if sooner). Each answer is
|
||||
// said in one line, with what it waited on and for how long (issue 1126: ace, g14 and shanks answered nothing for a
|
||||
// minute while nothing said why). An answer not composed within the bound is unready, naming what the node waits on.
|
||||
func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandidateAsk) delivery.ComposeCandidateAnswer {
|
||||
func (s *Shadow) ComposeCandidate(ctx context.Context, ask nodeengine.ComposeCandidateInput) nodeengine.ComposeCandidateAnswer {
|
||||
s.mu.Lock()
|
||||
c := s.c
|
||||
s.mu.Unlock()
|
||||
@@ -99,14 +102,14 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
|
||||
ctx, cancel := context.WithTimeout(ctx, candidateWithin)
|
||||
defer cancel()
|
||||
run := &candidateRun{}
|
||||
done := make(chan delivery.ComposeCandidateAnswer, 1)
|
||||
done := make(chan nodeengine.ComposeCandidateAnswer, 1)
|
||||
go func() { done <- s.composeCandidate(ctx, c, ask, began, run) }()
|
||||
var a delivery.ComposeCandidateAnswer
|
||||
var a nodeengine.ComposeCandidateAnswer
|
||||
select {
|
||||
case a = <-done:
|
||||
case <-ctx.Done():
|
||||
_, on := run.said(c.now())
|
||||
a = delivery.ComposeCandidateAnswer{Node: c.env.Node, Candidate: ask.Candidate, Verdict: delivery.ComposeCandidateUnready,
|
||||
a = nodeengine.ComposeCandidateAnswer{Node: c.env.Node, Candidate: ask.Candidate, Verdict: nec.Unready,
|
||||
Refused: fmt.Sprintf("the node did not compose the candidate within %s: it waits on %s", candidateWithin, on),
|
||||
Took: c.now().Sub(began).Milliseconds()}
|
||||
}
|
||||
@@ -121,19 +124,19 @@ func (s *Shadow) ComposeCandidate(ctx context.Context, ask delivery.ComposeCandi
|
||||
}
|
||||
|
||||
// composeCandidate composes ask on scratch copies of c, each added to run as it starts.
|
||||
func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery.ComposeCandidateAsk, began time.Time, run *candidateRun) delivery.ComposeCandidateAnswer {
|
||||
func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengine.ComposeCandidateInput, began time.Time, run *candidateRun) nodeengine.ComposeCandidateAnswer {
|
||||
node := c.env.Node
|
||||
a := delivery.ComposeCandidateAnswer{Node: node, Candidate: ask.Candidate}
|
||||
refuse := func(why string) delivery.ComposeCandidateAnswer {
|
||||
a.Verdict, a.Refused, a.Modules, a.Own = delivery.ComposeCandidateRefused, why, nil, delivery.ModuleComposed{}
|
||||
a := nodeengine.ComposeCandidateAnswer{Node: node, Candidate: ask.Candidate}
|
||||
refuse := func(why string) nodeengine.ComposeCandidateAnswer {
|
||||
a.Verdict, a.Refused, a.Modules, a.Own = nec.Refused, why, nil, nodeengine.Module{}
|
||||
a.Took = c.now().Sub(began).Milliseconds()
|
||||
return a
|
||||
}
|
||||
// A node that cannot compose now (its shadow not ready, a read the bus did not answer, the bound passed) answers
|
||||
// unready: the checker takes it as no answer (ADR 0593 §9) by the verdict, the words only saying why.
|
||||
unready := func(why string) delivery.ComposeCandidateAnswer {
|
||||
unready := func(why string) nodeengine.ComposeCandidateAnswer {
|
||||
a = refuse(why)
|
||||
a.Verdict = delivery.ComposeCandidateUnready
|
||||
a.Verdict = nec.Unready
|
||||
return a
|
||||
}
|
||||
group, heads, err := delivery.ParseCompositionKey(ask.Candidate)
|
||||
@@ -181,6 +184,7 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
if err != nil {
|
||||
return unready(err.Error())
|
||||
}
|
||||
sentSize := 0
|
||||
laidComps, laidHeld, laidErr := laid.composeShadow(ctx, running, touched, func(scope frombus.Scope, fps map[string]string) {
|
||||
if !ask.Input {
|
||||
return
|
||||
@@ -196,13 +200,19 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
in.ComposedHow.HeldBefore = maps.Clone(baseHeld)
|
||||
// As the sandbox reads it (composer.SentInput, the one shape): this node's own modules as taken in from their
|
||||
// builds, every other module's manifest left to the cases.
|
||||
// The seat's contract carries it as a JSON object; its numbers are kept as written (json.Number).
|
||||
if body, err := json.Marshal(composer.SentInput(in, laid.env.Node)); err == nil {
|
||||
a.Input = body
|
||||
d := json.NewDecoder(bytes.NewReader(body))
|
||||
d.UseNumber()
|
||||
var obj map[string]any
|
||||
if d.Decode(&obj) == nil {
|
||||
a.Input, sentSize = obj, len(body)
|
||||
}
|
||||
}
|
||||
})
|
||||
if len(a.Input) > maxSentInput {
|
||||
if sentSize > maxSentInput {
|
||||
// Past what the bus carries in one answer, the checker would hear nothing: refused, saying so.
|
||||
size := len(a.Input)
|
||||
size := sentSize
|
||||
a.Input = nil
|
||||
return refuse(fmt.Sprintf("the composition input this node composed from is %d bytes, more than the %d an answer on the bus carries", size, maxSentInput))
|
||||
}
|
||||
@@ -215,8 +225,8 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
return unready("a read on the bus did not answer: " + laidErr.Error())
|
||||
case laidErr != nil && baseErr != nil:
|
||||
// The node composes none of its own without the candidate either: the candidate changes nothing of that.
|
||||
a.Verdict, a.Modules = delivery.ComposeCandidateComposed, []delivery.ModuleComposed{}
|
||||
a.Own = delivery.ModuleComposed{Module: delivery.HaveNodeWord, State: delivery.ModuleHeldState, Words: laidErr.Error()}
|
||||
a.Verdict, a.Modules = nec.Composed, []nodeengine.Module{}
|
||||
a.Own = nodeengine.Module{Module: delivery.HaveNodeWord, State: nec.ModuleHeld, Words: laidErr.Error()}
|
||||
a.Took = c.now().Sub(began).Milliseconds()
|
||||
return a
|
||||
case laidErr != nil:
|
||||
@@ -230,15 +240,15 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
return out
|
||||
}
|
||||
was, now := byModule(baseComps), byModule(laidComps)
|
||||
one := func(module string) (delivery.ModuleComposed, bool) {
|
||||
m := delivery.ModuleComposed{Module: module}
|
||||
one := func(module string) (nodeengine.Module, bool) {
|
||||
m := nodeengine.Module{Module: module}
|
||||
if module == "" {
|
||||
m.Module = delivery.HaveNodeWord
|
||||
}
|
||||
if why, held := laidHeld[module]; held {
|
||||
m.State, m.Words = delivery.ModuleRefusedState, why
|
||||
m.State, m.Words = nec.ModuleRefused, why
|
||||
if _, before := baseHeld[module]; before || baseErr != nil {
|
||||
m.State = delivery.ModuleHeldState
|
||||
m.State = nec.ModuleHeld
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
@@ -248,10 +258,10 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
}
|
||||
dig, err := delivery.ComposedDigestOf(comp)
|
||||
if err != nil {
|
||||
m.State, m.Words = delivery.ModuleRefusedState, err.Error()
|
||||
m.State, m.Words = nec.ModuleRefused, err.Error()
|
||||
return m, true
|
||||
}
|
||||
m.State, m.FP, m.Digest = delivery.ModuleComposedState, comp.FP, dig
|
||||
m.State, m.Fp, m.Digest = nec.ModuleComposed, comp.FP, dig
|
||||
if before, ok := was[module]; !ok {
|
||||
m.Changed = true
|
||||
} else if d, err := delivery.ComposedDigestOf(before); err != nil || d != dig {
|
||||
@@ -273,24 +283,24 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask delivery
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
a.Modules = []delivery.ModuleComposed{}
|
||||
a.Modules = []nodeengine.Module{}
|
||||
refused := false
|
||||
for _, m := range names {
|
||||
if mc, ok := one(m); ok {
|
||||
a.Modules = append(a.Modules, mc)
|
||||
refused = refused || mc.State == delivery.ModuleRefusedState
|
||||
refused = refused || mc.State == nec.ModuleRefused
|
||||
}
|
||||
}
|
||||
if own, ok := one(""); ok {
|
||||
a.Own = own
|
||||
} else {
|
||||
a.Own = delivery.ModuleComposed{Module: delivery.HaveNodeWord, State: delivery.ModuleHeldState,
|
||||
a.Own = nodeengine.Module{Module: delivery.HaveNodeWord, State: nec.ModuleHeld,
|
||||
Words: "the node composes no composition of its own"}
|
||||
}
|
||||
refused = refused || a.Own.State == delivery.ModuleRefusedState
|
||||
a.Verdict = delivery.ComposeCandidateComposed
|
||||
refused = refused || a.Own.State == nec.ModuleRefused
|
||||
a.Verdict = nec.Composed
|
||||
if refused {
|
||||
a.Verdict = delivery.ComposeCandidateRefused
|
||||
a.Verdict = nec.Refused
|
||||
}
|
||||
a.Took = c.now().Sub(began).Milliseconds()
|
||||
return a
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
@@ -76,7 +78,7 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() { _ = s.Run(ctx) }()
|
||||
if a := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: accepted}); a.Verdict != delivery.ComposeCandidateUnready || a.Refused == "" {
|
||||
if a := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: accepted}); a.Verdict != nec.Unready || a.Refused == "" {
|
||||
t.Errorf("a shadow not ready answered %+v", a)
|
||||
}
|
||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||
@@ -98,45 +100,46 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
a := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: accepted})
|
||||
if _, err := delivery.EncodeComposeCandidateAnswer(a); err != nil {
|
||||
a := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: accepted})
|
||||
if _, err := nec.EncodeAnswer(a); err != nil {
|
||||
t.Fatalf("the answer does not fit the seat's shape: %v (%+v)", err, a)
|
||||
}
|
||||
if a.Verdict != delivery.ComposeCandidateComposed || len(a.Modules) != 1 || a.Modules[0].Module != "lemurs" ||
|
||||
a.Modules[0].State != delivery.ModuleComposedState || a.Modules[0].FP != fpNew || !a.Modules[0].Changed ||
|
||||
a.Own.Module != delivery.HaveNodeWord || a.Took > delivery.ComposeAnswerBound.Milliseconds() {
|
||||
if a.Verdict != nec.Composed || len(a.Modules) != 1 || a.Modules[0].Module != "lemurs" ||
|
||||
a.Modules[0].State != nec.ModuleComposed || a.Modules[0].Fp != fpNew || !a.Modules[0].Changed ||
|
||||
a.Own.Module != delivery.HaveNodeWord || a.Took > nec.AnswerBound.Milliseconds() {
|
||||
t.Errorf("an accepted candidate answered %+v", a)
|
||||
}
|
||||
// Issue 774: the input it sends the sandbox says how it composed the node (the builds), so the sandbox composes it
|
||||
// through the same path.
|
||||
sent := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: accepted, Input: true})
|
||||
sent := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: accepted, Input: true})
|
||||
var in compose.Input
|
||||
if err := json.Unmarshal(sent.Input, &in); err != nil || in.ComposedHow == nil || in.ComposedHow.Node != "n1" ||
|
||||
raw, _ := json.Marshal(sent.Input)
|
||||
if err := json.Unmarshal(raw, &in); err != nil || in.ComposedHow == nil || in.ComposedHow.Node != "n1" ||
|
||||
in.ComposedHow.Builds["lemurs"] != fpNew {
|
||||
t.Errorf("the input sent for the sandbox does not say how the node composes: %+v, %v", in.ComposedHow, err)
|
||||
}
|
||||
r := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: refusedKey})
|
||||
if r.Verdict != delivery.ComposeCandidateRefused || len(r.Modules) != 1 || r.Modules[0].State != delivery.ModuleRefusedState ||
|
||||
r := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: refusedKey})
|
||||
if r.Verdict != nec.Refused || len(r.Modules) != 1 || r.Modules[0].State != nec.ModuleRefused ||
|
||||
!strings.Contains(r.Modules[0].Words, "a field the judge refuses") {
|
||||
t.Errorf("a candidate whose manifest the indexer refused answered %+v", r)
|
||||
}
|
||||
if _, err := delivery.EncodeComposeCandidateAnswer(r); err != nil {
|
||||
if _, err := nec.EncodeAnswer(r); err != nil {
|
||||
t.Errorf("the refused answer does not fit the seat's shape: %v", err)
|
||||
}
|
||||
missing, _ := delivery.CompositionKey("novox/x#8", "0123456789ab")
|
||||
if m := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") {
|
||||
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") {
|
||||
t.Errorf("a candidate the indexer does not hold answered %+v", m)
|
||||
}
|
||||
// The group's candidate moved to other heads since the ask: the node composes not the one asked, and answers
|
||||
// unready (no answer), naming the heads it holds.
|
||||
moved, _ := delivery.CompositionKey("fix/940-no-retire-on-partial-read", "0123456789ab")
|
||||
if m := s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: moved}); m.Verdict != delivery.ComposeCandidateUnready ||
|
||||
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: moved}); m.Verdict != nec.Unready ||
|
||||
!strings.Contains(m.Refused, "heads") {
|
||||
t.Errorf("a candidate at other heads answered %+v", m)
|
||||
}
|
||||
ended, end := context.WithCancel(ctx)
|
||||
end()
|
||||
if e := s.ComposeCandidate(ended, delivery.ComposeCandidateAsk{Candidate: accepted}); e.Verdict != delivery.ComposeCandidateUnready {
|
||||
if e := s.ComposeCandidate(ended, nodeengine.ComposeCandidateInput{Candidate: accepted}); e.Verdict != nec.Unready {
|
||||
t.Errorf("an ended bound answered %+v", e)
|
||||
}
|
||||
if len(state.written()) != 0 || applied.Load() != 0 || handed.Load() != 0 {
|
||||
@@ -214,11 +217,11 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
||||
words string // in the answer's words
|
||||
said string // in the line said of the last answer
|
||||
}{
|
||||
{name: "one ask reads each manifest once", asks: 1, verdict: delivery.ComposeCandidateComposed, reads: 1,
|
||||
{name: "one ask reads each manifest once", asks: 1, verdict: nec.Composed, reads: 1,
|
||||
said: "compose-candidate fix/1126."},
|
||||
{name: "a later ask reads none again", asks: 2, verdict: delivery.ComposeCandidateComposed, reads: 1,
|
||||
{name: "a later ask reads none again", asks: 2, verdict: nec.Composed, reads: 1,
|
||||
said: "read(s) waited on"},
|
||||
{name: "a read that does not answer is named at the bound", stuck: true, asks: 1, verdict: delivery.ComposeCandidateUnready,
|
||||
{name: "a read that does not answer is named at the bound", stuck: true, asks: 1, verdict: nec.Unready,
|
||||
words: "waits on", said: "answered unready"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -240,9 +243,9 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
||||
return get(rctx, bucket, k)
|
||||
}
|
||||
}
|
||||
var a delivery.ComposeCandidateAnswer
|
||||
var a nodeengine.ComposeCandidateAnswer
|
||||
for range tc.asks {
|
||||
a = s.ComposeCandidate(ctx, delivery.ComposeCandidateAsk{Candidate: key})
|
||||
a = s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: key})
|
||||
}
|
||||
if a.Verdict != tc.verdict || !strings.Contains(a.Refused, tc.words) {
|
||||
t.Errorf("answered %s %q; want %s with %q", a.Verdict, a.Refused, tc.verdict, tc.words)
|
||||
|
||||
@@ -218,6 +218,13 @@ type Pass struct {
|
||||
Earlier bool `json:"earlier,omitempty"`
|
||||
}
|
||||
|
||||
// LastHeld is what the last pass held, each module with why; nil before a pass or when it held none.
|
||||
func (c *Composer) LastHeld() map[string]string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return maps.Clone(c.last.Held)
|
||||
}
|
||||
|
||||
// askedWithin is how recent a pass answers an ask of the verb compose without a new one: the first wait after a seat
|
||||
// that did not answer (laterAgain's first).
|
||||
var askedWithin = 5 * time.Second
|
||||
@@ -915,6 +922,13 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
}
|
||||
c.paid()
|
||||
pass.Held = held
|
||||
// The provisioner client's account the node runs with, dropped by this pass, is said: every filled write is refused
|
||||
// under a declaration that names none (filling.JudgeAccount), and no module is handed to it (issue 774, ace,
|
||||
// 2026-11-04 01:29). A placed client keeps its account while it is held (compose.Node.RunningProvisioner).
|
||||
if was, now := ownProvisioner(running), ownProvisioner(mapValues(compositions)); was != "" && now == "" && nodeWanted {
|
||||
c.sayOnceEach(fmt.Sprintf("the node's own composition names no provisioner client's account, where what runs names %s: "+
|
||||
"once applied, every filled write is refused and no module is handed to the provisioner", was))
|
||||
}
|
||||
// A module refused alone is held: it keeps what it runs, its have says why, and every other module is applied (issue
|
||||
// 1126: one module's refusal froze the whole node).
|
||||
done := c.waiting("the agent marking each held module's have")
|
||||
@@ -1699,3 +1713,21 @@ func (c *Composer) ComposedToday(ctx context.Context, module, fp string) (string
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ownProvisioner is the provisioner client's account the node's own composition among cs names, or "".
|
||||
func ownProvisioner(cs []delivery.Composed) string {
|
||||
for _, c := range cs {
|
||||
if c.Module == "" {
|
||||
return c.Own.Provisioner
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func mapValues(m map[string]delivery.Composed) []delivery.Composed {
|
||||
out := make([]delivery.Composed, 0, len(m))
|
||||
for _, k := range slices.Sorted(maps.Keys(m)) {
|
||||
out = append(out, m[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What a node knows is written by one writer at a time, completely or not at all, and never loses a record without
|
||||
// saying so (novox/hq issue 1585). On novox, 2026-11-04, the node-engine's state had lost the records of two archives
|
||||
// (nats's program snapshot and its tools bundle) that a save of 2026-10-24 had written into its temporary file and
|
||||
// never renamed into place, and about 260 such temporary files (~80 MB) lay beside it: the process ended between
|
||||
// writing a save and renaming it, again and again, because the engine stopped without waiting for the apply in flight
|
||||
// (cmd/mesh-host, stopBetweenApplies). The temporary files and the lost records are the same fault.
|
||||
|
||||
// tempPattern is the name a save writes before it renames the file into place.
|
||||
const tempPattern = ".state-*.json"
|
||||
|
||||
// saving is the one writer of this process: a save holds it from its read of what was saved before to its rename and
|
||||
// the directory's flush, and Quiesce holds it for good.
|
||||
var saving sync.Mutex
|
||||
|
||||
// Said is where a save says loudly what must not pass in silence: a record the state held before and this save would
|
||||
// drop although nothing forgot it. The journal when the engine runs as a service.
|
||||
var Said = func(line string) { fmt.Fprintln(os.Stderr, line) }
|
||||
|
||||
// Save writes the state, atomically and durably: the whole file to a temporary one beside it, flushed, renamed over
|
||||
// the old one, and the directory flushed so the rename itself survives a power cut.
|
||||
//
|
||||
// Atomic because the alternative has a failure mode with no floor: a host interrupted while
|
||||
// writing loses the record of everything it owns, and then owns nothing it can clean up.
|
||||
//
|
||||
// Before it writes, it reads what was saved before and says, through Said, each resource or hold that state recorded
|
||||
// and this one would drop although this copy never forgot it (Forget, Release): a save from a copy read before
|
||||
// another writer saved, or a copy that lost a record on the way, is said at once rather than found weeks later when a
|
||||
// file the host put there reads as foreign. The save still goes ahead: it is what the apply found on the machine, and
|
||||
// refusing it would lose that too.
|
||||
func Save(path string, s State) error {
|
||||
saving.Lock()
|
||||
defer saving.Unlock()
|
||||
return save(path, s)
|
||||
}
|
||||
|
||||
func save(path string, s State) error {
|
||||
s.UpdatedAt = time.Now().UTC()
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("making room for the node's state: %w", err)
|
||||
}
|
||||
|
||||
if before, err := Load(path); err == nil {
|
||||
for _, lost := range Dropped(before, s) {
|
||||
Said(fmt.Sprintf("the node's state is saved without the record of %s, which the state saved before held "+
|
||||
"and nothing forgot: what it put on this machine is no longer known as the mesh's (issue 1585)", lost))
|
||||
}
|
||||
}
|
||||
|
||||
raw, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding the node's state: %w", err)
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
|
||||
tmp, err := os.CreateTemp(dir, tempPattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing the node's state: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
|
||||
if _, err := tmp.Write(raw); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("writing the node's state: %w", err)
|
||||
}
|
||||
// Flushed before the rename: a rename is atomic, and a rename of a file whose contents are
|
||||
// still in the page cache is atomically the wrong thing.
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("flushing the node's state: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("closing the node's state: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o600); err != nil {
|
||||
return fmt.Errorf("securing the node's state: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
return fmt.Errorf("replacing the node's state: %w", err)
|
||||
}
|
||||
// And the directory, so the new name is on disk and not only the new contents.
|
||||
if d, err := os.Open(dir); err == nil {
|
||||
syncErr := d.Sync()
|
||||
d.Close()
|
||||
if syncErr != nil {
|
||||
return fmt.Errorf("flushing the directory of the node's state: %w", syncErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Dropped is each resource and hold before recorded that after does not, and that after never forgot: "resource <id>"
|
||||
// or "hold <id>", sorted.
|
||||
func Dropped(before, after State) []string {
|
||||
kept := map[string]bool{}
|
||||
for _, r := range after.Resources {
|
||||
kept["resource "+r.ID] = true
|
||||
}
|
||||
for _, h := range after.Held {
|
||||
kept["hold "+h.ID] = true
|
||||
}
|
||||
var out []string
|
||||
for _, r := range before.Resources {
|
||||
if k := "resource " + r.ID; !kept[k] && !after.forgotten[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
for _, h := range before.Held {
|
||||
if k := "hold " + h.ID; !kept[k] && !after.forgotten[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// forget notes that this copy dropped a record on purpose.
|
||||
func (s *State) forget(what string) {
|
||||
if s.forgotten == nil {
|
||||
s.forgotten = map[string]bool{}
|
||||
}
|
||||
s.forgotten[what] = true
|
||||
}
|
||||
|
||||
// Quiesce waits for a save in flight to end and lets no other start in this process, for the engine's stop: a process
|
||||
// that ends while a save writes leaves a temporary file and the state before it (issue 1585). It returns what lets saves
|
||||
// start again, for a test; a stopping engine never calls it.
|
||||
func Quiesce() (release func()) {
|
||||
saving.Lock()
|
||||
return saving.Unlock
|
||||
}
|
||||
|
||||
// RemoveLeftovers removes the temporary files of saves that never reached their rename, beside the state at path, and
|
||||
// names each it removed. Under the machine's apply lock, so no save of another process is in flight: every writer of
|
||||
// the state takes it (Lock).
|
||||
func RemoveLeftovers(path string) ([]string, error) {
|
||||
unlock, err := Lock(path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
saving.Lock()
|
||||
defer saving.Unlock()
|
||||
found, err := filepath.Glob(filepath.Join(filepath.Dir(path), tempPattern))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var removed []string
|
||||
var failed []string
|
||||
for _, f := range found {
|
||||
if err := os.Remove(f); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", filepath.Base(f), err))
|
||||
continue
|
||||
}
|
||||
removed = append(removed, filepath.Base(f))
|
||||
}
|
||||
sort.Strings(removed)
|
||||
if len(failed) > 0 {
|
||||
return removed, fmt.Errorf("leftover saves of the node's state could not be removed: %s", strings.Join(failed, "; "))
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
+11
-49
@@ -16,7 +16,6 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -263,6 +262,10 @@ type State struct {
|
||||
// Members is every library group the engine put a tool runner's account in that it was not in before (ADR 0328
|
||||
// §11, as ADR 0252 records an account's groups): taken back when no longer declared.
|
||||
Members []Membership `json:"members,omitempty"`
|
||||
|
||||
// forgotten is what this copy dropped on purpose since it was loaded (Forget, Release): never written, read by Save
|
||||
// to tell a record given up from one lost (issue 1585).
|
||||
forgotten map[string]bool
|
||||
}
|
||||
|
||||
// MadeGroup is one library group the engine made.
|
||||
@@ -484,7 +487,7 @@ func (s *State) RecordHeld(h Held) {
|
||||
|
||||
// Release drops a hold, once its module is taken and the host has converged what was held.
|
||||
func (s *State) Release(id string) {
|
||||
kept := s.Held[:0]
|
||||
kept := make([]Held, 0, len(s.Held))
|
||||
for _, h := range s.Held {
|
||||
if h.ID != id {
|
||||
kept = append(kept, h)
|
||||
@@ -494,6 +497,7 @@ func (s *State) Release(id string) {
|
||||
if len(s.Held) == 0 {
|
||||
s.Held = nil
|
||||
}
|
||||
s.forget("hold " + id)
|
||||
}
|
||||
|
||||
// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one.
|
||||
@@ -563,51 +567,6 @@ func Load(path string) (State, error) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Save writes the state, atomically.
|
||||
//
|
||||
// Atomic because the alternative has a failure mode with no floor: a host interrupted while
|
||||
// writing loses the record of everything it owns, and then owns nothing it can clean up.
|
||||
func Save(path string, s State) error {
|
||||
s.UpdatedAt = time.Now().UTC()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return fmt.Errorf("making room for the node's state: %w", err)
|
||||
}
|
||||
|
||||
raw, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding the node's state: %w", err)
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing the node's state: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
|
||||
if _, err := tmp.Write(raw); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("writing the node's state: %w", err)
|
||||
}
|
||||
// Flushed before the rename: a rename is atomic, and a rename of a file whose contents are
|
||||
// still in the page cache is atomically the wrong thing.
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("flushing the node's state: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("closing the node's state: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o600); err != nil {
|
||||
return fmt.Errorf("securing the node's state: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
return fmt.Errorf("replacing the node's state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Record adds or replaces what is known about one resource, preserving order.
|
||||
func (s *State) Record(a Applied) {
|
||||
for i, existing := range s.Resources {
|
||||
@@ -640,15 +599,18 @@ func FormerID(id, target string) string { return id + "@former:" + target }
|
||||
// IsFormer says whether a record names a former target.
|
||||
func IsFormer(id string) bool { return strings.Contains(id, "@former:") }
|
||||
|
||||
// Forget drops a resource from what the node owns.
|
||||
// Forget drops a resource from what the node owns, and notes that it was dropped on purpose, so the save that follows
|
||||
// does not say it lost the record (Save). A new slice, never the old one's array: a copy of the state taken before keeps
|
||||
// what it held.
|
||||
func (s *State) Forget(id string) {
|
||||
kept := s.Resources[:0]
|
||||
kept := make([]Applied, 0, len(s.Resources))
|
||||
for _, r := range s.Resources {
|
||||
if r.ID != id {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
s.Resources = kept
|
||||
s.forget("resource " + id)
|
||||
}
|
||||
|
||||
// Orphans returns what the host applied and the declaration no longer names, newest first.
|
||||
|
||||
@@ -93,6 +93,66 @@ func TestSavingLeavesNoDebrisBehind(t *testing.T) {
|
||||
}
|
||||
t.Errorf("expected only the state file, found: %v", names)
|
||||
}
|
||||
|
||||
// Issue 1585: a save cut off between its write and its rename (the engine ended under it) left its temporary file;
|
||||
// the engine's start removes them under the apply lock and names each, and leaves the state and every other file.
|
||||
t.Run("leftovers of cut-off saves are removed", func(t *testing.T) {
|
||||
for _, name := range []string{".state-3758449435.json", ".state-1403853615.json", ".declared-1.json"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("{}"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
removed, err := RemoveLeftovers(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(removed, ",") != ".state-1403853615.json,.state-3758449435.json" {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
for _, keep := range []string{"state.json", ".declared-1.json"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, keep)); err != nil {
|
||||
t.Errorf("%s was not left: %v", keep, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Issue 1585: a save that drops a record the state saved before held, which nothing forgot, says so loudly; one
|
||||
// forgotten or released on purpose is not said. Forget never writes into the array a copy taken before still reads.
|
||||
t.Run("a dropped record is said, a forgotten one is not", func(t *testing.T) {
|
||||
var said []string
|
||||
was := Said
|
||||
Said = func(line string) { said = append(said, line) }
|
||||
defer func() { Said = was }()
|
||||
before := State{Resources: []Applied{{ID: "c", Type: "file", Target: "/c"},
|
||||
{ID: "nats.program-snapshot", Type: "archive", Target: "/s"}, {ID: "nats.bundle-tools", Type: "archive", Target: "/t"}},
|
||||
Held: []Held{{ID: "h"}}}
|
||||
if err := Save(path, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "resource a,") {
|
||||
t.Errorf("the record of a, saved before and dropped, was not said: %v", said)
|
||||
}
|
||||
said = nil
|
||||
copied := before
|
||||
copied.Forget("c")
|
||||
copied.Release("h")
|
||||
if before.Resources[0].ID != "c" {
|
||||
t.Errorf("Forget wrote into the copy taken before: %+v", before.Resources)
|
||||
}
|
||||
if err := Save(path, copied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Errorf("a record forgotten on purpose was said lost: %v", said)
|
||||
}
|
||||
stale := State{Resources: []Applied{{ID: "nats.bundle-tools", Type: "archive", Target: "/t"}}}
|
||||
if err := Save(path, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "resource nats.program-snapshot") {
|
||||
t.Errorf("a lost record was not said, or said wrong: %v", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRecordReplacesRatherThanDuplicating(t *testing.T) {
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -38,7 +38,9 @@ BACKOFF="${MESH_HOST_BACKOFF:-5}"
|
||||
# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.
|
||||
BOUND="${MESH_HOST_TRIAL_BOUND:-600}"
|
||||
TICK="${MESH_HOST_TRIAL_TICK:-5}"
|
||||
GRACE="${MESH_HOST_STOP_GRACE:-20}"
|
||||
# How long a host told to stop is waited for before it is killed, in seconds: past the 60 s the host itself waits for
|
||||
# the apply in flight to end and save (novox/hq issue 1585), so a stop never cuts a save off.
|
||||
GRACE="${MESH_HOST_STOP_GRACE:-75}"
|
||||
ONCE="${MESH_HOST_RUN_ONCE:-}" # tests run one iteration; nothing else sets this
|
||||
|
||||
ATTEMPTS="$STATE_DIR/start-attempts"
|
||||
|
||||
@@ -5,4 +5,6 @@
|
||||
name="nox-mesh-host"
|
||||
command="/usr/lib/nox-mesh-host/launch"
|
||||
supervisor="supervise-daemon"
|
||||
# The same stop bound as the systemd unit's TimeoutStopSec (issue 1585): TERM, 120 s, then KILL.
|
||||
retry="TERM/120/KILL/5"
|
||||
depend() { need net; }
|
||||
|
||||
@@ -12,6 +12,9 @@ Restart=always
|
||||
RestartSec=5s
|
||||
StateDirectory=mesh-host
|
||||
KillMode=mixed
|
||||
# Past the launcher's own stop grace (75 s), which is past the 60 s the host waits for the apply in
|
||||
# flight to end and save (novox/hq issue 1585): the service manager's kill is the last resort.
|
||||
TimeoutStopSec=120
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -75,6 +75,13 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ADR 0614 §4: the node-engine's own manifest uses only words every running engine reads (the baseline, what an engine
|
||||
// stating none is read as), so its new build is never held for a word only it reads.
|
||||
if needs, err := catalogue.ManifestNeeds(raw); err != nil || len(needs) == 0 {
|
||||
t.Fatalf("the manifest's needs: %v, %v", needs, err)
|
||||
} else if missing := catalogue.MissingManifestReads(needs, nil); len(missing) != 0 {
|
||||
t.Errorf("the manifest needs %v, which an engine stating nothing does not read: its new build would be held", missing)
|
||||
}
|
||||
// 774 (the alert table): the state health, one key per node, the node's own health report, kept until
|
||||
// written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time).
|
||||
// ADR 0577 clause 1: and the state data, one key per module placed on the node that declares data items, kept until
|
||||
|
||||
Reference in New Issue
Block a user