Say a machine waiting for its push as waiting, not uncomposable (hq issue 275)

Between assign and push a module's own secrets are not made yet; D1 composed
without making them and raised an urgent 'nothing can be sent' that the next
push resolved silently. D1 now composes as the push would (Foreseeing): a
secret the push makes gets a stand-in and is named, one the push is refused on
is refused with the push's words. Waiting is said only past 30 minutes, as a
warning. D3 and D13 expect a holder only once its machine was sent it and
reported or had ten minutes to.
This commit is contained in:
jochen
2026-10-06 18:05:38 +02:00
parent 2b5060789f
commit dcee8cb5bf
10 changed files with 567 additions and 45 deletions
+281
View File
@@ -0,0 +1,281 @@
package main
import (
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
func aKeeper() catalogue.Manifest {
return catalogue.Manifest{Module: "keeper", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
}}
}
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
// are made — without a bus to carry it.
func pushedBy(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, node, plan, settings)
if err != nil {
t.Fatal(err)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
digest := digestOf(body)
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
t.Fatal(err)
}
return digest
}
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
func pushedAndApplied(t *testing.T, open *stores, node string) {
t.Helper()
digest := pushedBy(t, open, node)
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
t.Fatal(err)
}
}
// d1 runs D1 once.
func d1(t *testing.T, open *stores) []conditions.Observation {
t.Helper()
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
return got
}
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
// can be compared about a secret that does not exist (status), and nothing here string-matches.
plan, settings, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
_, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading)
var notMade *catalogue.NotMadeError
if !errors.As(err, &notMade) || notMade.Module != "keeper" || notMade.Name != "repository" {
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
}
// Foreseen, it composes, names what the push makes, and made nothing.
foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing)
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
}
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a machine waiting for a push raised %+v", got)
}
// Past the bound: a warning, not urgent, saying what the push will make.
before := awaitingPushBound
awaitingPushBound = -time.Minute
t.Cleanup(func() { awaitingPushBound = before })
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
}
// Pushed: the secret is made and D1 says nothing.
pushedBy(t, open, "laptop")
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a pushed machine still raised %+v", got)
}
}
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aTalker())
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
}
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
// uncomposable for that — the stand-in hides nothing.
register(t, open, aKeeper())
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
t.Fatal(err)
}
one, two := rivals()
register(t, open, one)
register(t, open, two)
_, _ = assign(ctx, open, "anchor", "rival-one")
_, _ = assign(ctx, open, "anchor", "rival-two")
keys := map[string]conditions.Severity{}
for _, o := range d1(t, open) {
keys[o.Key()] = o.Severity
}
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
t.Fatalf("a real failure beside a waiting secret: %v", keys)
}
}
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
// refused on it, so D1 is too.
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
t.Fatalf("a given secret under an old key: %+v", got)
}
}
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
sent := time.Now()
since, err := open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.After(sent) {
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
since, err = open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.Before(sent.Add(-time.Second)) {
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
}
}
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
// it and had time to report — never between assign and push.
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
now := time.Now()
sent := now.Add(-time.Minute)
long := now.Add(-time.Hour)
cases := []struct {
name string
send lastSend
want bool
}{
{"never sent", lastSend{}, false},
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
}
for _, c := range cases {
if got := c.send.settled("keeper", now); got != c.want {
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
}
}
}
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
sends, err := readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("a holder assigned and not pushed is expected to answer")
}
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
t.Fatal("a module the machine was sent long ago is not expected to answer")
}
pushedBy(t, open, "laptop")
sends, err = readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
}
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
t.Fatal("pushed past the grace, a holder is not expected")
}
if _, ok := sends["laptop"].carried["keeper"]; !ok {
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
}
pushedAndApplied(t, open, "laptop")
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
}
}
+14 -6
View File
@@ -121,15 +121,15 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen
return out, nil
}
// declaredOn is every data item a machine's composition declares, and whether something there holds
// node-backup to measure them.
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) {
// declaredOn is every data item a machine's composition declares, and the module there that holds
// node-backup to measure them — empty for none.
func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) {
var out []inventory.DeclaredData
held := false
held := ""
for _, m := range plan.Modules {
for _, c := range m.Claims {
if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat {
held = true
held = m.Module
}
}
for _, it := range m.DataItems() {
@@ -166,6 +166,10 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
}
heard := heardMachines(d)
now := time.Now()
delivered, err := readDeliveries(ctx, open.inventory)
if err != nil {
return nil, err
}
type machine struct {
name string
@@ -185,7 +189,11 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error)
// declaring nothing: retiring its data on that would be acting on an unreadable result.
continue
}
declared, held := declaredOn(plan)
declared, holder := declaredOn(plan)
// **A holder is asked only once its machine has been sent it and had time to report** (novox/hq
// issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it
// measured" about it was a warning for a push nobody had made yet.
held := holder != "" && delivered[n.Name].settled(holder, now)
machines = append(machines, &machine{name: n.Name, declared: declared, held: held})
}
// Every holder asked at once, as D8 asks every ban list.
+4
View File
@@ -299,6 +299,10 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
t.Fatal(err)
}
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
for _, node := range []string{"laptop", "anchor"} {
pushedAndApplied(t, open, node)
}
conn := onATestBus(t)
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
+2 -1
View File
@@ -75,7 +75,8 @@ type probe struct {
// probe added to a design is a row added here.
var probeRegistry = []probe{
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation",
From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations},
From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1,
run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " +
+56 -23
View File
@@ -378,13 +378,19 @@ func declarationFor(ctx context.Context, open *stores, node string,
// So the mesh chooses a port when it commits to sending one, and every other caller reads what
// was chosen. A module with nothing assigned yet has never been sent, which is exactly what a
// machine "waiting" means — the read needs no number to be right about that.
type Choosing bool
type Choosing int
const (
// Allocating is the send path: what is not assigned yet is assigned now and kept.
Allocating Choosing = true
// Reading is every question: what is assigned is used, and nothing is created.
Reading Choosing = false
Reading Choosing = iota
// Allocating is the send path: what is not assigned yet is assigned now and kept.
Allocating
// Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing
// is created, and an own secret the next send WOULD make is composed with a stand-in and named
// (sendable.foreseen) rather than failing the composition — while one the send would be refused
// (a bus credential nobody issued, a given secret under an old key) is refused here as it would
// be there. Never sent: the stand-in is not a sealed value.
Foreseeing
)
func declarationWith(ctx context.Context, open *stores, node string,
@@ -406,7 +412,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
out := sendable{Resources: composed.Resources, Adoption: adoption,
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
unbound: with.Unbound}
unbound: with.Unbound, foreseen: composed.Foreseen}
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
// 0221). Read only on the send path: a question about what would be sent records nothing.
if choosing == Allocating {
@@ -507,6 +513,30 @@ func reportLeftOut(node string, declared sendable) {
}
}
// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued.
//
// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is
// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted
// by `module issue` and sealed by it; a push that made a random one would deliver a file the process
// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on
// a question asked ahead of it (Foreseeing), so that one is never told the push will make it.
func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error {
if name != "broker" {
return nil
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return err
} else if !minted {
return fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
module, node, user, module, node)
}
return nil
}
// renderingFor is everything a node's declaration is composed with, and the node's record.
func renderingFor(ctx context.Context, open *stores, node string,
plan catalogue.Resolution, settings catalogue.SettingsBy,
@@ -524,7 +554,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
// consumer is told are all derived from it.
// What this machine was already given, for a composition that may not allocate.
already := map[string]map[int]int{}
if choosing == Reading {
if choosing != Allocating {
held, err := inv.PortsFor(ctx, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
@@ -610,6 +640,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
// And each module's own secrets — a superuser password, an administrator, an account. Made
// per node, so a module running on three machines has three.
needed := map[string]map[string]string{}
foreseen := map[string]map[string]bool{}
for _, m := range plan.Modules {
for name := range m.OwnSecrets {
// Minted on the send path and only read on every other. Making one is an insert, and
@@ -617,27 +648,29 @@ func renderingFor(ctx context.Context, open *stores, node string,
var sealed string
var err error
if choosing == Allocating {
// **The broker credential is never invented here** (novox/hq issue 203). Every other
// own secret is the mesh's to make — a password nobody else knows — but this one
// is an account on the bus, minted by `module issue` and sealed by it; a push that
// made a random one would deliver a file the process cannot read and report the
// machine applied. Refused by name, with the verb.
if name == "broker" {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
} else if !minted {
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
"%s on %s has no bus credential: nothing was issued for %s, and a push "+
"would seal a placeholder its process cannot read (novox/hq issue 203). "+
"`module issue %s --node %s`, then push again",
m.Module, node, user, m.Module, node)
}
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
sealed, err = inv.SecretForModule(ctx, node, m.Module, name)
} else {
var held bool
sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name)
if err == nil && !held && choosing == Foreseeing {
// Asked ahead of the send: what the send would do about it, by the send's own
// rules. Made by it — a stand-in, named; refused by it — refused here, the same
// words (novox/hq issue 275).
if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
if foreseen[m.Module] == nil {
foreseen[m.Module] = map[string]bool{}
}
foreseen[m.Module][name] = true
continue
}
if err == nil && !held {
// Never issued, so this machine cannot be running it. Left out rather than
// invented: an empty string here would compose a declaration that differs
@@ -829,7 +862,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
}
return catalogue.Rendering{
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Machines: machines, Zones: zones, Holders: replicas,
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
+101 -3
View File
@@ -23,6 +23,7 @@ import (
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/overlay"
@@ -32,8 +33,25 @@ import (
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
// look" (ADR 0227 rule 4).
// awaitingPushBound is how long a machine may have something only its next push makes — an own
// secret of a module assigned since its last push — before that is said (novox/hq issue 275).
var awaitingPushBound = 30 * time.Minute
// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound.
const kindAwaitingPush = "awaiting-push"
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
// (mesh-host's `validate`, the package the host runs) takes it.
//
// **Composed as the next push would compose it, without making anything** (Foreseeing, novox/hq issue
// 275). Between `assign` and `push` a module's own secrets are not made yet — the push makes them —
// and a composition that only reads them failed, which raised an urgent "nothing can be sent" about a
// machine the next push sent to without a word. So a secret the push WILL make is composed with a
// stand-in and named; one the push would be refused on is refused here, with the push's words. A
// machine whose declaration composes and validates with only such stand-ins is waiting for a push,
// not broken: nothing is said until awaitingPushBound passes from when it began waiting, and then a
// warning (`awaiting-push`) naming what the push will make — never urgent, because nothing is wrong
// that a push does not fix.
func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
open := d.open
nodes, err := open.inventory.Nodes(ctx)
@@ -59,10 +77,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
if err != nil && !unresolvable(err) {
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
}
var problems []string
var problems, foreseen []string
if err == nil && gensErr == nil {
var declared sendable
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil {
foreseen = declared.foreseen
// With the order it was last sent, so the validator reads the envelope a machine is sent
// — its epoch included (novox/hq to-be 45 §6).
var serr error
@@ -94,11 +113,30 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
n.Name, len(problems), problems[0]),
Said: strings.Join(problems, "; ")})
case len(foreseen) > 0:
since, err := open.inventory.AwaitingSince(ctx, n.Name)
if err != nil {
return nil, err // the store, not the machine: the probe could not run
}
if waited := time.Since(since); waited > awaitingPushBound {
out = append(out, awaitingPush(n.Name, foreseen, since, waited))
}
}
}
return out, nil
}
// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has
// made what it names, past awaitingPushBound (novox/hq issue 275).
func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation {
made := strings.Join(foreseen, ", ")
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush,
Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+
"`push %s` sends it", node, waited.Round(time.Minute), made, node),
Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)}
}
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
// takes as final (issue 262).
@@ -221,6 +259,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
return nil, err
}
heard := heardMachines(d)
delivered, err := readDeliveries(ctx, d.open.inventory)
if err != nil {
return nil, err
}
now := time.Now()
expected := map[string]map[string]bool{} // seat → machine
add := func(seat, node string) {
if expected[seat] == nil {
@@ -248,7 +291,10 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
continue
}
for _, node := range e.On {
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) {
// Assigned is not there yet: a holder is expected once its machine was sent it and
// had time to report, never between `assign` and `push` (novox/hq issue 275).
if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) &&
delivered[node].settled(e.Manifest.Module, now) {
add(s.Name, node)
}
}
@@ -277,6 +323,58 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err
return sortedFound(out), nil
}
// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report
// before what the declaration carries is expected to answer (novox/hq issue 275).
var reportGrace = 10 * time.Minute
// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send
// carried, when, and whether the machine has reported acting on it.
type lastSend struct {
// carried is the modules its last send carried; nil when that was not kept (a send from before
// it was, or one by hand), and then every module is taken as carried — as before this existed.
carried map[string]string
// sent is when its current declaration went to it; nil if nothing ever did.
sent *time.Time
// current says its last report names the declaration last sent.
current bool
}
// settled says whether a module's holder on this machine can be asked to answer now: the machine was
// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to.
// A machine never sent anything holds nothing the mesh put there.
func (d lastSend) settled(module string, now time.Time) bool {
if d.sent == nil {
return false
}
if d.carried != nil {
if _, in := d.carried[module]; !in {
return false
}
}
return d.current || now.Sub(*d.sent) > reportGrace
}
// readDeliveries is every machine's last send, by name, from the records a send and a report keep.
func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) {
reports, err := inv.LastReports(ctx)
if err != nil {
return nil, err
}
out := map[string]lastSend{}
for _, r := range reports {
builds, known, err := inv.SentBuilds(ctx, r.Node)
if err != nil {
return nil, err
}
d := lastSend{sent: r.Sent, current: r.Current}
if known {
d.carried = builds
}
out[r.Node] = d
}
return out, nil
}
// heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw.
func heardMachines(d *doctor) map[string]bool {
out := map[string]bool{}
+4
View File
@@ -54,6 +54,10 @@ type sendable struct {
// unbound is every consumer whose credential from this machine is on record and that is bound
// elsewhere (novox/hq issue 274); for push and plan to say, never on the wire.
unbound []catalogue.Unbound
// foreseen is every own secret composed with a stand-in, as `module/name`: what the next send will
// make (Foreseeing, novox/hq issue 275). Never on the wire, and a declaration that has any is never
// sent.
foreseen []string
// Builds is the build of each module this declaration carries — module to the commit its build
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
// Composed only on the send path; nil records that it is not known.
+39 -5
View File
@@ -87,6 +87,12 @@ type Rendering struct {
// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
// name the module gave it.
Needed map[string]map[string]string
// Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a
// composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in
// its place and lists it in Composed.Foreseen, rather than failing for a value only a send
// makes. Nil on every composition that is sent, and on every other question, which then refuse
// a secret not made with a *NotMadeError.
Foreseen map[string]map[string]bool
// Mesh is every node's address on the private network, which is what a rule saying "from the
// mesh" resolves to. Passed in for the same reason grants are: who else is on the network is
@@ -275,6 +281,25 @@ type Composed struct {
// compose. Its held things are kept and its containers untouched — the machine is told so —
// and it is told everything else.
LeftOut map[string]string
// Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen),
// as `module/name`, sorted: what the next send will make. Never set on a declaration that is
// sent — a placeholder in a sealed file is a credential the process cannot read.
Foreseen []string
}
// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of
// the send. Not a sealed value: nothing composed with it may be sent.
const ForeseenSealed = "foreseen: made by the next send"
// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275):
// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that
// fails for any other reason without reading the words.
type NotMadeError struct {
Module, Name string
}
func (e *NotMadeError) Error() string {
return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name)
}
// LeftOut is which of this machine's modules a declaration composed with these settings leaves
@@ -295,10 +320,12 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
owner := map[string]string{}
received := map[string]map[string][]Contribution{}
leftOut := map[string]string{}
resources, err := r.compose(with, owner, received, leftOut)
var foreseen []string
resources, err := r.compose(with, owner, received, leftOut, &foreseen)
if err != nil {
return Composed{}, err
}
sort.Strings(foreseen)
if with.BusMembership != "" {
// The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a
// secret and placed where the host looks for exactly this (design 28, task 5.2).
@@ -307,7 +334,8 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
"sealed": with.BusMembership, "mode": "0600",
})
}
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut}, nil
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut,
Foreseen: foreseen}, nil
}
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
@@ -317,7 +345,8 @@ func BusMembershipID() string { return "bus-membership" }
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
func (r Resolution) compose(with Rendering, owner map[string]string,
received map[string]map[string][]Contribution, leftOut map[string]string) ([]map[string]any, error) {
received map[string]map[string][]Contribution, leftOut map[string]string,
foreseen *[]string) ([]map[string]any, error) {
// **A setting is judged where it is stored, and an impossible one costs a module, not a
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
// this module uncomposable; it is left out of the declaration — its held things kept, its
@@ -534,12 +563,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
}
for _, name := range sortedKeys(m.OwnSecrets) {
sealed := with.Needed[m.Module][name]
if sealed == "" && with.Foreseen[m.Module][name] {
// Not made, and the next send makes it: composed with a stand-in so that whatever
// else this composition would refuse is still found (novox/hq issue 275).
sealed = ForeseenSealed
*foreseen = append(*foreseen, m.Module+"/"+name)
}
if sealed == "" {
// Declared and not made. Refused rather than skipped: a module whose own
// credential is silently absent starts, fails to authenticate, and the reason is
// three layers away from the machine reporting it.
return nil, fmt.Errorf(
"%s needs a secret called %q and none was made for it", m.Module, name)
return nil, &NotMadeError{Module: m.Module, Name: name}
}
// The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175,
// to-be 38 WP3): its process is composed `user: <account>` where the node has one, and a
+19
View File
@@ -976,6 +976,25 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
return err
}
// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue
// 275): the oldest assignment on it made after it was last sent, or — when nothing was assigned since —
// when it was last sent, or when it joined if it never was. The moment a bound on "not pushed yet" is
// read from: every change a push carries happened after the last push, and an assignment is the one
// that says when.
func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) {
var since time.Time
err := i.store.Pool().QueryRow(ctx,
`select coalesce(
(select min(a.assigned) from assignment a
where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)),
n.sent_at, n.created)
from node n where n.name = $1`, name).Scan(&since)
if errors.Is(err, pgx.ErrNoRows) {
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
}
return since, err
}
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
// last sent before it was, sent a declaration by hand, or one the mesh does not know.
+47 -7
View File
@@ -327,9 +327,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
return "", err
}
if key == "" {
return "", fmt.Errorf(
"%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
module, node)
return "", noSealingKey(module, node)
}
record, err := i.NodeByName(ctx, node)
if err != nil {
@@ -349,10 +347,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
// would put 32 random bytes where a working credential was: the machine would apply it,
// report success, and whatever reads it would fail to authenticate somewhere else
// entirely — with the mesh insisting the secret was delivered, which it was.
return "", fmt.Errorf(
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
"since generated a new sealing key. The mesh cannot make another; issue it again",
module, node, name, node)
return "", acceptedUnderAnOldKey(module, node, name)
}
operator, err := i.OperatorKey(ctx)
@@ -381,6 +376,51 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
return made.ForConsumer, nil
}
// WouldMakeSecretForModule says what SecretForModule would do about an own secret, without doing it:
// nil when it would make one (or one is held), and otherwise the very refusal it would meet. The read
// a question asked ahead of a send uses (novox/hq issue 275), so that "the next push makes this" is
// told apart from "the next push fails on this" by the same rules the push applies.
func (i *Inventory) WouldMakeSecretForModule(ctx context.Context, node, module, name string) error {
key, err := i.SealingKeyOf(ctx, node)
if err != nil {
return err
}
if key == "" {
return noSealingKey(module, node)
}
record, err := i.NodeByName(ctx, node)
if err != nil {
return err
}
var against, origin string
err = i.store.Pool().QueryRow(ctx,
`select node_key, origin from module_secret where node = $1 and module = $2 and name = $3`,
record.ID, module, name).Scan(&against, &origin)
switch {
case errors.Is(err, pgx.ErrNoRows):
return nil
case err != nil:
return err
case against != key && origin == "accepted":
return acceptedUnderAnOldKey(module, node, name)
}
return nil
}
// noSealingKey is the refusal for a secret on a machine that has no key to seal it to.
func noSealingKey(module, node string) error {
return fmt.Errorf("%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
module, node)
}
// acceptedUnderAnOldKey is the refusal for a given secret sealed to a key the machine no longer has.
func acceptedUnderAnOldKey(module, node, name string) error {
return fmt.Errorf(
"%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+
"since generated a new sealing key. The mesh cannot make another; issue it again",
module, node, name, node)
}
// AcceptSecretForModule keeps a value somebody supplied as a module's own secret.
//
// The counterpart to SecretForModule, which generates one. Some of what a module needs the mesh