Say a machine waiting for its push as waiting, not uncomposable (hq issue 275)
Between assign and push a module's own secrets are not made yet; D1 composed without making them and raised an urgent 'nothing can be sent' that the next push resolved silently. D1 now composes as the push would (Foreseeing): a secret the push makes gets a stand-in and is named, one the push is refused on is refused with the push's words. Waiting is said only past 30 minutes, as a warning. D3 and D13 expect a holder only once its machine was sent it and reported or had ten minutes to.
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -121,15 +121,15 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and whether something there holds
|
||||
// node-backup to measure them.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
|
||||
// declaredOn is every data item a machine's composition declares, and the module there that holds
|
||||
// node-backup to measure them — empty for none.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
|
||||
var out []inventory.DeclaredData
|
||||
held := false
|
||||
held := ""
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = true
|
||||
held = m.Module
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
@@ -166,6 +166,10 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
delivered, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
@@ -185,7 +189,11 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, held := declaredOn(plan)
|
||||
declared, holder := declaredOn(plan)
|
||||
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
|
||||
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
|
||||
// measured" about it was a warning for a push nobody had made yet.
|
||||
held := holder != "" && delivered[n.Name].settled(holder, now)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
|
||||
@@ -299,6 +299,10 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||
|
||||
@@ -75,7 +75,8 @@ type probe struct {
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
|
||||
+56
-23
@@ -378,13 +378,19 @@ func declarationFor(ctx context.Context, open *stores, node string,
|
||||
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
|
||||
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
|
||||
// machine "waiting" means — the read needs no number to be right about that.
|
||||
type Choosing bool
|
||||
type Choosing int
|
||||
|
||||
const (
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating Choosing = true
|
||||
// Reading is every question: what is assigned is used, and nothing is created.
|
||||
Reading Choosing = false
|
||||
Reading Choosing = iota
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating
|
||||
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
|
||||
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
|
||||
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
|
||||
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
|
||||
// be there. Never sent: the stand-in is not a sealed value.
|
||||
Foreseeing
|
||||
)
|
||||
|
||||
func declarationWith(ctx context.Context, open *stores, node string,
|
||||
@@ -406,7 +412,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound}
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -507,6 +513,30 @@ func reportLeftOut(node string, declared sendable) {
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
//
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
|
||||
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
|
||||
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
|
||||
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
|
||||
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
|
||||
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
|
||||
if name != "broker" {
|
||||
return nil
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return err
|
||||
} else if !minted {
|
||||
return fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
module, node, user, module, node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
@@ -524,7 +554,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// consumer is told are all derived from it.
|
||||
// What this machine was already given, for a composition that may not allocate.
|
||||
already := map[string]map[int]int{}
|
||||
if choosing == Reading {
|
||||
if choosing != Allocating {
|
||||
held, err := inv.PortsFor(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
@@ -610,6 +640,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// And each module's own secrets — a superuser password, an administrator, an account. Made
|
||||
// per node, so a module running on three machines has three.
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
for name := range m.OwnSecrets {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
@@ -617,27 +648,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other
|
||||
// own secret is the mesh's to make — a password nobody else knows — but this one
|
||||
// is an account on the bus, minted by `module issue` and sealed by it; a push that
|
||||
// made a random one would deliver a file the process cannot read and report the
|
||||
// machine applied. Refused by name, with the verb.
|
||||
if name == "broker" {
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
} else if !minted {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
m.Module, node, user, m.Module, node)
|
||||
}
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
|
||||
if err == nil && !held && choosing == Foreseeing {
|
||||
// Asked ahead of the send: what the send would do about it, by the send's own
|
||||
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
|
||||
// words (novox/hq issue 275).
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if foreseen[m.Module] == nil {
|
||||
foreseen[m.Module] = map[string]bool{}
|
||||
}
|
||||
foreseen[m.Module][name] = true
|
||||
continue
|
||||
}
|
||||
if err == nil && !held {
|
||||
// Never issued, so this machine cannot be running it. Left out rather than
|
||||
// invented: an empty string here would compose a declaration that differs
|
||||
@@ -829,7 +862,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
@@ -32,8 +33,25 @@ import (
|
||||
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||
// look" (ADR 0227 rule 4).
|
||||
|
||||
// awaitingPushBound is how long a machine may have something only its next push makes — an own
|
||||
// secret of a module assigned since its last push — before that is said (novox/hq issue 275).
|
||||
var awaitingPushBound = 30 * time.Minute
|
||||
|
||||
// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound.
|
||||
const kindAwaitingPush = "awaiting-push"
|
||||
|
||||
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||
//
|
||||
// **Composed as the next push would compose it, without making anything** (Foreseeing, novox/hq issue
|
||||
// 275). Between `assign` and `push` a module's own secrets are not made yet — the push makes them —
|
||||
// and a composition that only reads them failed, which raised an urgent "nothing can be sent" about a
|
||||
// machine the next push sent to without a word. So a secret the push WILL make is composed with a
|
||||
// stand-in and named; one the push would be refused on is refused here, with the push's words. A
|
||||
// machine whose declaration composes and validates with only such stand-ins is waiting for a push,
|
||||
// not broken: nothing is said until awaitingPushBound passes from when it began waiting, and then a
|
||||
// warning (`awaiting-push`) naming what the push will make — never urgent, because nothing is wrong
|
||||
// that a push does not fix.
|
||||
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
open := d.open
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
@@ -59,10 +77,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
if err != nil && !unresolvable(err) {
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems []string
|
||||
var problems, foreseen []string
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||
foreseen = declared.foreseen
|
||||
// With the order it was last sent, so the validator reads the envelope a machine is sent
|
||||
// — its epoch included (novox/hq to-be 45 §6).
|
||||
var serr error
|
||||
@@ -94,11 +113,30 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||
n.Name, len(problems), problems[0]),
|
||||
Said: strings.Join(problems, "; ")})
|
||||
case len(foreseen) > 0:
|
||||
since, err := open.inventory.AwaitingSince(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err // the store, not the machine: the probe could not run
|
||||
}
|
||||
if waited := time.Since(since); waited > awaitingPushBound {
|
||||
out = append(out, awaitingPush(n.Name, foreseen, since, waited))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has
|
||||
// made what it names, past awaitingPushBound (novox/hq issue 275).
|
||||
func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation {
|
||||
made := strings.Join(foreseen, ", ")
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush,
|
||||
Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+
|
||||
"`push %s` sends it", node, waited.Round(time.Minute), made, node),
|
||||
Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)}
|
||||
}
|
||||
|
||||
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||
// takes as final (issue 262).
|
||||
@@ -221,6 +259,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
delivered, err := readDeliveries(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
@@ -248,7 +291,10 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
||||
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
||||
delivered[node].settled(e.Manifest.Module, now) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
@@ -277,6 +323,58 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
|
||||
// before what the declaration carries is expected to answer (novox/hq issue 275).
|
||||
var reportGrace = 10 * time.Minute
|
||||
|
||||
// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send
|
||||
// carried, when, and whether the machine has reported acting on it.
|
||||
type lastSend struct {
|
||||
// carried is the modules its last send carried; nil when that was not kept (a send from before
|
||||
// it was, or one by hand), and then every module is taken as carried — as before this existed.
|
||||
carried map[string]string
|
||||
// sent is when its current declaration went to it; nil if nothing ever did.
|
||||
sent *time.Time
|
||||
// current says its last report names the declaration last sent.
|
||||
current bool
|
||||
}
|
||||
|
||||
// settled says whether a module's holder on this machine can be asked to answer now: the machine was
|
||||
// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to.
|
||||
// A machine never sent anything holds nothing the mesh put there.
|
||||
func (d lastSend) settled(module string, now time.Time) bool {
|
||||
if d.sent == nil {
|
||||
return false
|
||||
}
|
||||
if d.carried != nil {
|
||||
if _, in := d.carried[module]; !in {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return d.current || now.Sub(*d.sent) > reportGrace
|
||||
}
|
||||
|
||||
// readDeliveries is every machine's last send, by name, from the records a send and a report keep.
|
||||
func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]lastSend{}
|
||||
for _, r := range reports {
|
||||
builds, known, err := inv.SentBuilds(ctx, r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := lastSend{sent: r.Sent, current: r.Current}
|
||||
if known {
|
||||
d.carried = builds
|
||||
}
|
||||
out[r.Node] = d
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||
func heardMachines(d *doctor) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
|
||||
@@ -54,6 +54,10 @@ type sendable struct {
|
||||
// unbound is every consumer whose credential from this machine is on record and that is bound
|
||||
// elsewhere (novox/hq issue 274); for push and plan to say, never on the wire.
|
||||
unbound []catalogue.Unbound
|
||||
// foreseen is every own secret composed with a stand-in, as `module/name`: what the next send will
|
||||
// make (Foreseeing, novox/hq issue 275). Never on the wire, and a declaration that has any is never
|
||||
// sent.
|
||||
foreseen []string
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
|
||||
@@ -87,6 +87,12 @@ type Rendering struct {
|
||||
// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
|
||||
// name the module gave it.
|
||||
Needed map[string]map[string]string
|
||||
// Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a
|
||||
// composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in
|
||||
// its place and lists it in Composed.Foreseen, rather than failing for a value only a send
|
||||
// makes. Nil on every composition that is sent, and on every other question, which then refuse
|
||||
// a secret not made with a *NotMadeError.
|
||||
Foreseen map[string]map[string]bool
|
||||
|
||||
// Mesh is every node's address on the private network, which is what a rule saying "from the
|
||||
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
|
||||
@@ -275,6 +281,25 @@ type Composed struct {
|
||||
// compose. Its held things are kept and its containers untouched — the machine is told so —
|
||||
// and it is told everything else.
|
||||
LeftOut map[string]string
|
||||
// Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen),
|
||||
// as `module/name`, sorted: what the next send will make. Never set on a declaration that is
|
||||
// sent — a placeholder in a sealed file is a credential the process cannot read.
|
||||
Foreseen []string
|
||||
}
|
||||
|
||||
// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of
|
||||
// the send. Not a sealed value: nothing composed with it may be sent.
|
||||
const ForeseenSealed = "foreseen: made by the next send"
|
||||
|
||||
// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275):
|
||||
// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that
|
||||
// fails for any other reason without reading the words.
|
||||
type NotMadeError struct {
|
||||
Module, Name string
|
||||
}
|
||||
|
||||
func (e *NotMadeError) Error() string {
|
||||
return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name)
|
||||
}
|
||||
|
||||
// LeftOut is which of this machine's modules a declaration composed with these settings leaves
|
||||
@@ -295,10 +320,12 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
owner := map[string]string{}
|
||||
received := map[string]map[string][]Contribution{}
|
||||
leftOut := map[string]string{}
|
||||
resources, err := r.compose(with, owner, received, leftOut)
|
||||
var foreseen []string
|
||||
resources, err := r.compose(with, owner, received, leftOut, &foreseen)
|
||||
if err != nil {
|
||||
return Composed{}, err
|
||||
}
|
||||
sort.Strings(foreseen)
|
||||
if with.BusMembership != "" {
|
||||
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
|
||||
// secret and placed where the host looks for exactly this (design 28, task 5.2).
|
||||
@@ -307,7 +334,8 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
"sealed": with.BusMembership, "mode": "0600",
|
||||
})
|
||||
}
|
||||
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut}, nil
|
||||
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut,
|
||||
Foreseen: foreseen}, nil
|
||||
}
|
||||
|
||||
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
|
||||
@@ -317,7 +345,8 @@ func BusMembershipID() string { return "bus-membership" }
|
||||
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
|
||||
|
||||
func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
received map[string]map[string][]Contribution, leftOut map[string]string) ([]map[string]any, error) {
|
||||
received map[string]map[string][]Contribution, leftOut map[string]string,
|
||||
foreseen *[]string) ([]map[string]any, error) {
|
||||
// **A setting is judged where it is stored, and an impossible one costs a module, not a
|
||||
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
|
||||
// this module uncomposable; it is left out of the declaration — its held things kept, its
|
||||
@@ -534,12 +563,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
}
|
||||
for _, name := range sortedKeys(m.OwnSecrets) {
|
||||
sealed := with.Needed[m.Module][name]
|
||||
if sealed == "" && with.Foreseen[m.Module][name] {
|
||||
// Not made, and the next send makes it: composed with a stand-in so that whatever
|
||||
// else this composition would refuse is still found (novox/hq issue 275).
|
||||
sealed = ForeseenSealed
|
||||
*foreseen = append(*foreseen, m.Module+"/"+name)
|
||||
}
|
||||
if sealed == "" {
|
||||
// Declared and not made. Refused rather than skipped: a module whose own
|
||||
// credential is silently absent starts, fails to authenticate, and the reason is
|
||||
// three layers away from the machine reporting it.
|
||||
return nil, fmt.Errorf(
|
||||
"%s needs a secret called %q and none was made for it", m.Module, name)
|
||||
return nil, &NotMadeError{Module: m.Module, Name: name}
|
||||
}
|
||||
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
|
||||
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
|
||||
|
||||
@@ -976,6 +976,25 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
|
||||
return err
|
||||
}
|
||||
|
||||
// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue
|
||||
// 275): the oldest assignment on it made after it was last sent, or — when nothing was assigned since —
|
||||
// when it was last sent, or when it joined if it never was. The moment a bound on "not pushed yet" is
|
||||
// read from: every change a push carries happened after the last push, and an assignment is the one
|
||||
// that says when.
|
||||
func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) {
|
||||
var since time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select coalesce(
|
||||
(select min(a.assigned) from assignment a
|
||||
where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)),
|
||||
n.sent_at, n.created)
|
||||
from node n where n.name = $1`, name).Scan(&since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
return since, err
|
||||
}
|
||||
|
||||
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
|
||||
// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
|
||||
// last sent before it was, sent a declaration by hand, or one the mesh does not know.
|
||||
|
||||
@@ -327,9 +327,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
|
||||
return "", err
|
||||
}
|
||||
if key == "" {
|
||||
return "", fmt.Errorf(
|
||||
"%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
|
||||
module, node)
|
||||
return "", noSealingKey(module, node)
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
@@ -349,10 +347,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
|
||||
// would put 32 random bytes where a working credential was: the machine would apply it,
|
||||
// report success, and whatever reads it would fail to authenticate somewhere else
|
||||
// entirely — with the mesh insisting the secret was delivered, which it was.
|
||||
return "", fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; issue it again",
|
||||
module, node, name, node)
|
||||
return "", acceptedUnderAnOldKey(module, node, name)
|
||||
}
|
||||
|
||||
operator, err := i.OperatorKey(ctx)
|
||||
@@ -381,6 +376,51 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
|
||||
return made.ForConsumer, nil
|
||||
}
|
||||
|
||||
// WouldMakeSecretForModule says what SecretForModule would do about an own secret, without doing it:
|
||||
// nil when it would make one (or one is held), and otherwise the very refusal it would meet. The read
|
||||
// a question asked ahead of a send uses (novox/hq issue 275), so that "the next push makes this" is
|
||||
// told apart from "the next push fails on this" by the same rules the push applies.
|
||||
func (i *Inventory) WouldMakeSecretForModule(ctx context.Context, node, module, name string) error {
|
||||
key, err := i.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
return noSealingKey(module, node)
|
||||
}
|
||||
record, err := i.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var against, origin string
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select node_key, origin from module_secret where node = $1 and module = $2 and name = $3`,
|
||||
record.ID, module, name).Scan(&against, &origin)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
case against != key && origin == "accepted":
|
||||
return acceptedUnderAnOldKey(module, node, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// noSealingKey is the refusal for a secret on a machine that has no key to seal it to.
|
||||
func noSealingKey(module, node string) error {
|
||||
return fmt.Errorf("%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
|
||||
module, node)
|
||||
}
|
||||
|
||||
// acceptedUnderAnOldKey is the refusal for a given secret sealed to a key the machine no longer has.
|
||||
func acceptedUnderAnOldKey(module, node, name string) error {
|
||||
return fmt.Errorf(
|
||||
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
|
||||
"since generated a new sealing key. The mesh cannot make another; issue it again",
|
||||
module, node, name, node)
|
||||
}
|
||||
|
||||
// AcceptSecretForModule keeps a value somebody supplied as a module's own secret.
|
||||
//
|
||||
// The counterpart to SecretForModule, which generates one. Some of what a module needs the mesh
|
||||
|
||||
Reference in New Issue
Block a user