Say a machine waiting for its push as waiting, not uncomposable (hq issue 275)
Between assign and push a module's own secrets are not made yet; D1 composed without making them and raised an urgent 'nothing can be sent' that the next push resolved silently. D1 now composes as the push would (Foreseeing): a secret the push makes gets a stand-in and is named, one the push is refused on is refused with the push's words. Waiting is said only past 30 minutes, as a warning. D3 and D13 expect a holder only once its machine was sent it and reported or had ten minutes to.
This commit is contained in:
jochen
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user