Say a machine waiting for its push as waiting, not uncomposable (hq issue 275)
Between assign and push a module's own secrets are not made yet; D1 composed without making them and raised an urgent 'nothing can be sent' that the next push resolved silently. D1 now composes as the push would (Foreseeing): a secret the push makes gets a stand-in and is named, one the push is refused on is refused with the push's words. Waiting is said only past 30 minutes, as a warning. D3 and D13 expect a holder only once its machine was sent it and reported or had ten minutes to.
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -121,15 +121,15 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaredOn is every data item a machine's composition declares, and whether something there holds
|
||||
// node-backup to measure them.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
|
||||
// declaredOn is every data item a machine's composition declares, and the module there that holds
|
||||
// node-backup to measure them — empty for none.
|
||||
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
|
||||
var out []inventory.DeclaredData
|
||||
held := false
|
||||
held := ""
|
||||
for _, m := range plan.Modules {
|
||||
for _, c := range m.Claims {
|
||||
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
|
||||
held = true
|
||||
held = m.Module
|
||||
}
|
||||
}
|
||||
for _, it := range m.DataItems() {
|
||||
@@ -166,6 +166,10 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
now := time.Now()
|
||||
delivered, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type machine struct {
|
||||
name string
|
||||
@@ -185,7 +189,11 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
|
||||
// declaring nothing: retiring its data on that would be acting on an unreadable result.
|
||||
continue
|
||||
}
|
||||
declared, held := declaredOn(plan)
|
||||
declared, holder := declaredOn(plan)
|
||||
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
|
||||
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
|
||||
// measured" about it was a warning for a push nobody had made yet.
|
||||
held := holder != "" && delivered[n.Name].settled(holder, now)
|
||||
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
|
||||
}
|
||||
// Every holder asked at once, as D8 asks every ban list.
|
||||
|
||||
@@ -299,6 +299,10 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||
|
||||
@@ -75,7 +75,8 @@ type probe struct {
|
||||
// probe added to a design is a row added here.
|
||||
var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
|
||||
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
|
||||
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
|
||||
|
||||
+56
-23
@@ -378,13 +378,19 @@ func declarationFor(ctx context.Context, open *stores, node string,
|
||||
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
|
||||
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
|
||||
// machine "waiting" means — the read needs no number to be right about that.
|
||||
type Choosing bool
|
||||
type Choosing int
|
||||
|
||||
const (
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating Choosing = true
|
||||
// Reading is every question: what is assigned is used, and nothing is created.
|
||||
Reading Choosing = false
|
||||
Reading Choosing = iota
|
||||
// Allocating is the send path: what is not assigned yet is assigned now and kept.
|
||||
Allocating
|
||||
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
|
||||
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
|
||||
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
|
||||
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
|
||||
// be there. Never sent: the stand-in is not a sealed value.
|
||||
Foreseeing
|
||||
)
|
||||
|
||||
func declarationWith(ctx context.Context, open *stores, node string,
|
||||
@@ -406,7 +412,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
|
||||
unbound: with.Unbound}
|
||||
unbound: with.Unbound, foreseen: composed.Foreseen}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
@@ -507,6 +513,30 @@ func reportLeftOut(node string, declared sendable) {
|
||||
}
|
||||
}
|
||||
|
||||
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
|
||||
//
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
|
||||
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
|
||||
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
|
||||
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
|
||||
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
|
||||
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
|
||||
if name != "broker" {
|
||||
return nil
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return err
|
||||
} else if !minted {
|
||||
return fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
module, node, user, module, node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
@@ -524,7 +554,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// consumer is told are all derived from it.
|
||||
// What this machine was already given, for a composition that may not allocate.
|
||||
already := map[string]map[int]int{}
|
||||
if choosing == Reading {
|
||||
if choosing != Allocating {
|
||||
held, err := inv.PortsFor(ctx, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
@@ -610,6 +640,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
// And each module's own secrets — a superuser password, an administrator, an account. Made
|
||||
// per node, so a module running on three machines has three.
|
||||
needed := map[string]map[string]string{}
|
||||
foreseen := map[string]map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
for name := range m.OwnSecrets {
|
||||
// Minted on the send path and only read on every other. Making one is an insert, and
|
||||
@@ -617,27 +648,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
var sealed string
|
||||
var err error
|
||||
if choosing == Allocating {
|
||||
// **The broker credential is never invented here** (novox/hq issue 203). Every other
|
||||
// own secret is the mesh's to make — a password nobody else knows — but this one
|
||||
// is an account on the bus, minted by `module issue` and sealed by it; a push that
|
||||
// made a random one would deliver a file the process cannot read and report the
|
||||
// machine applied. Refused by name, with the verb.
|
||||
if name == "broker" {
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
} else if !minted {
|
||||
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
|
||||
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
|
||||
"`module issue %s --node %s`, then push again",
|
||||
m.Module, node, user, m.Module, node)
|
||||
}
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
|
||||
} else {
|
||||
var held bool
|
||||
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
|
||||
if err == nil && !held && choosing == Foreseeing {
|
||||
// Asked ahead of the send: what the send would do about it, by the send's own
|
||||
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
|
||||
// words (novox/hq issue 275).
|
||||
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
if foreseen[m.Module] == nil {
|
||||
foreseen[m.Module] = map[string]bool{}
|
||||
}
|
||||
foreseen[m.Module][name] = true
|
||||
continue
|
||||
}
|
||||
if err == nil && !held {
|
||||
// Never issued, so this machine cannot be running it. Left out rather than
|
||||
// invented: an empty string here would compose a declaration that differs
|
||||
@@ -829,7 +862,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
@@ -32,8 +33,25 @@ import (
|
||||
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
|
||||
// look" (ADR 0227 rule 4).
|
||||
|
||||
// awaitingPushBound is how long a machine may have something only its next push makes — an own
|
||||
// secret of a module assigned since its last push — before that is said (novox/hq issue 275).
|
||||
var awaitingPushBound = 30 * time.Minute
|
||||
|
||||
// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound.
|
||||
const kindAwaitingPush = "awaiting-push"
|
||||
|
||||
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
|
||||
// (mesh-host's `validate`, the package the host runs) takes it.
|
||||
//
|
||||
// **Composed as the next push would compose it, without making anything** (Foreseeing, novox/hq issue
|
||||
// 275). Between `assign` and `push` a module's own secrets are not made yet — the push makes them —
|
||||
// and a composition that only reads them failed, which raised an urgent "nothing can be sent" about a
|
||||
// machine the next push sent to without a word. So a secret the push WILL make is composed with a
|
||||
// stand-in and named; one the push would be refused on is refused here, with the push's words. A
|
||||
// machine whose declaration composes and validates with only such stand-ins is waiting for a push,
|
||||
// not broken: nothing is said until awaitingPushBound passes from when it began waiting, and then a
|
||||
// warning (`awaiting-push`) naming what the push will make — never urgent, because nothing is wrong
|
||||
// that a push does not fix.
|
||||
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
open := d.open
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
@@ -59,10 +77,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
if err != nil && !unresolvable(err) {
|
||||
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
|
||||
}
|
||||
var problems []string
|
||||
var problems, foreseen []string
|
||||
if err == nil && gensErr == nil {
|
||||
var declared sendable
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
|
||||
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
|
||||
foreseen = declared.foreseen
|
||||
// With the order it was last sent, so the validator reads the envelope a machine is sent
|
||||
// — its epoch included (novox/hq to-be 45 §6).
|
||||
var serr error
|
||||
@@ -94,11 +113,30 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
|
||||
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
|
||||
n.Name, len(problems), problems[0]),
|
||||
Said: strings.Join(problems, "; ")})
|
||||
case len(foreseen) > 0:
|
||||
since, err := open.inventory.AwaitingSince(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, err // the store, not the machine: the probe could not run
|
||||
}
|
||||
if waited := time.Since(since); waited > awaitingPushBound {
|
||||
out = append(out, awaitingPush(n.Name, foreseen, since, waited))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has
|
||||
// made what it names, past awaitingPushBound (novox/hq issue 275).
|
||||
func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation {
|
||||
made := strings.Join(foreseen, ", ")
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush,
|
||||
Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+
|
||||
"`push %s` sends it", node, waited.Round(time.Minute), made, node),
|
||||
Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)}
|
||||
}
|
||||
|
||||
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
|
||||
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
|
||||
// takes as final (issue 262).
|
||||
@@ -221,6 +259,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
return nil, err
|
||||
}
|
||||
heard := heardMachines(d)
|
||||
delivered, err := readDeliveries(ctx, d.open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
expected := map[string]map[string]bool{} // seat → machine
|
||||
add := func(seat, node string) {
|
||||
if expected[seat] == nil {
|
||||
@@ -248,7 +291,10 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
continue
|
||||
}
|
||||
for _, node := range e.On {
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
|
||||
// Assigned is not there yet: a holder is expected once its machine was sent it and
|
||||
// had time to report, never between `assign` and `push` (novox/hq issue 275).
|
||||
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
|
||||
delivered[node].settled(e.Manifest.Module, now) {
|
||||
add(s.Name, node)
|
||||
}
|
||||
}
|
||||
@@ -277,6 +323,58 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
return sortedFound(out), nil
|
||||
}
|
||||
|
||||
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
|
||||
// before what the declaration carries is expected to answer (novox/hq issue 275).
|
||||
var reportGrace = 10 * time.Minute
|
||||
|
||||
// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send
|
||||
// carried, when, and whether the machine has reported acting on it.
|
||||
type lastSend struct {
|
||||
// carried is the modules its last send carried; nil when that was not kept (a send from before
|
||||
// it was, or one by hand), and then every module is taken as carried — as before this existed.
|
||||
carried map[string]string
|
||||
// sent is when its current declaration went to it; nil if nothing ever did.
|
||||
sent *time.Time
|
||||
// current says its last report names the declaration last sent.
|
||||
current bool
|
||||
}
|
||||
|
||||
// settled says whether a module's holder on this machine can be asked to answer now: the machine was
|
||||
// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to.
|
||||
// A machine never sent anything holds nothing the mesh put there.
|
||||
func (d lastSend) settled(module string, now time.Time) bool {
|
||||
if d.sent == nil {
|
||||
return false
|
||||
}
|
||||
if d.carried != nil {
|
||||
if _, in := d.carried[module]; !in {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return d.current || now.Sub(*d.sent) > reportGrace
|
||||
}
|
||||
|
||||
// readDeliveries is every machine's last send, by name, from the records a send and a report keep.
|
||||
func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]lastSend{}
|
||||
for _, r := range reports {
|
||||
builds, known, err := inv.SentBuilds(ctx, r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := lastSend{sent: r.Sent, current: r.Current}
|
||||
if known {
|
||||
d.carried = builds
|
||||
}
|
||||
out[r.Node] = d
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
|
||||
func heardMachines(d *doctor) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
|
||||
@@ -54,6 +54,10 @@ type sendable struct {
|
||||
// unbound is every consumer whose credential from this machine is on record and that is bound
|
||||
// elsewhere (novox/hq issue 274); for push and plan to say, never on the wire.
|
||||
unbound []catalogue.Unbound
|
||||
// foreseen is every own secret composed with a stand-in, as `module/name`: what the next send will
|
||||
// make (Foreseeing, novox/hq issue 275). Never on the wire, and a declaration that has any is never
|
||||
// sent.
|
||||
foreseen []string
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
|
||||
Reference in New Issue
Block a user