diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go new file mode 100644 index 0000000..e0890c6 --- /dev/null +++ b/cmd/mesh-controller/awaiting_push_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/overlay" +) + +// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed +// the machine's declaration without making anything, failed on the missing secret, and raised an urgent +// "nothing can be sent to " — seen live on 2026-10-06, a desktop notification for a machine +// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a +// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning. + +// aKeeper is a module with an own secret the mesh makes — a backup repository's password. +func aKeeper() catalogue.Manifest { + return catalogue.Manifest{Module: "keeper", Version: "1", + OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}}, + Resources: []map[string]any{ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"}, + }} +} + +// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets +// are made — without a bus to carry it. +func pushedBy(t *testing.T, open *stores, node string) string { + t.Helper() + ctx := t.Context() + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, node, plan, settings) + if err != nil { + t.Fatal(err) + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + digest := digestOf(body) + if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil { + t.Fatal(err) + } + return digest +} + +// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration. +func pushedAndApplied(t *testing.T, open *stores, node string) { + t.Helper() + digest := pushedBy(t, open, node) + record, err := open.inventory.NodeByName(t.Context(), node) + if err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{ + Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil { + t.Fatal(err) + } +} + +// d1 runs D1 once. +func d1(t *testing.T, open *stores) []conditions.Observation { + t.Helper() + got, err := probeDeclarations(t.Context(), &doctor{open: open}) + if err != nil { + t.Fatal(err) + } + return got +} + +// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is +// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing. +func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + + // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing + // can be compared about a secret that does not exist (status), and nothing here string-matches. + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + _, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading) + var notMade *catalogue.NotMadeError + if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { + t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) + } + // Foreseen, it composes, names what the push makes, and made nothing. + foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing) + if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { + t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { + t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) + } + + // Within the bound: nothing at all — no urgent, no warning, nobody notified. + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a machine waiting for a push raised %+v", got) + } + + // Past the bound: a warning, not urgent, saying what the push will make. + before := awaitingPushBound + awaitingPushBound = -time.Minute + t.Cleanup(func() { awaitingPushBound = before }) + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || + !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { + t.Fatalf("a machine left un-pushed past the bound: %+v", got) + } + + // Pushed: the secret is made and D1 says nothing. + pushedBy(t, open, "laptop") + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a pushed machine still raised %+v", got) + } +} + +// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make. +// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words. +func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aTalker()) + if _, err := assign(ctx, open, "laptop", "talker"); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "module issue talker --node laptop") { + t.Fatalf("a push that will be refused was not said urgently: %+v", got) + } + + // And a machine whose composition fails for anything else, with a secret waiting beside it, is + // uncomposable for that — the stand-in hides nothing. + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { + t.Fatal(err) + } + one, two := rivals() + register(t, open, one) + register(t, open, two) + _, _ = assign(ctx, open, "anchor", "rival-one") + _, _ = assign(ctx, open, "anchor", "rival-two") + keys := map[string]conditions.Severity{} + for _, o := range d1(t, open) { + keys[o.Key()] = o.Severity + } + if keys["machine.anchor.uncomposable"] != conditions.Urgent { + t.Fatalf("a real failure beside a waiting secret: %v", keys) + } +} + +// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is +// refused on it, so D1 is too. +func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { + t.Fatalf("a given secret under an old key: %+v", got) + } +} + +// The bound is read from when the machine began waiting: the oldest assignment since its last send. +func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + sent := time.Now() + since, err := open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.After(sent) { + t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) + } + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + since, err = open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.Before(sent.Add(-time.Second)) { + t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) + } +} + +// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent +// it and had time to report — never between assign and push. +func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) { + now := time.Now() + sent := now.Add(-time.Minute) + long := now.Add(-time.Hour) + cases := []struct { + name string + send lastSend + want bool + }{ + {"never sent", lastSend{}, false}, + {"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false}, + {"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false}, + {"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true}, + {"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true}, + {"sent before builds were kept", lastSend{sent: &long, current: true}, true}, + } + for _, c := range cases { + if got := c.send.settled("keeper", now); got != c.want { + t.Errorf("%s: settled %v, want %v", c.name, got, c.want) + } + } +} + +// And through the stores: assigned, not in the last send; pushed and reported, carried and settled. +func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + sends, err := readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("a holder assigned and not pushed is expected to answer") + } + if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { + t.Fatal("a module the machine was sent long ago is not expected to answer") + } + pushedBy(t, open, "laptop") + sends, err = readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("pushed a moment ago and not reported, a holder is already expected") + } + if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { + t.Fatal("pushed past the grace, a holder is not expected") + } + if _, ok := sends["laptop"].carried["keeper"]; !ok { + t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) + } + pushedAndApplied(t, open, "laptop") + if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { + t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) + } +} diff --git a/cmd/mesh-controller/data.go b/cmd/mesh-controller/data.go index 2399382..3615caa 100644 --- a/cmd/mesh-controller/data.go +++ b/cmd/mesh-controller/data.go @@ -121,15 +121,15 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen return out, nil } -// declaredOn is every data item a machine's composition declares, and whether something there holds -// node-backup to measure them. -func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, bool) { +// declaredOn is every data item a machine's composition declares, and the module there that holds +// node-backup to measure them — empty for none. +func declaredOn(plan catalogue.Resolution) ([]inventory.DeclaredData, string) { var out []inventory.DeclaredData - held := false + held := "" for _, m := range plan.Modules { for _, c := range m.Claims { if s, known := catalogue.SeatNamed(c.Name); known && s.Name == catalogue.BackupSeat { - held = true + held = m.Module } } for _, it := range m.DataItems() { @@ -166,6 +166,10 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) } heard := heardMachines(d) now := time.Now() + delivered, err := readDeliveries(ctx, open.inventory) + if err != nil { + return nil, err + } type machine struct { name string @@ -185,7 +189,11 @@ func probeData(ctx context.Context, d *doctor) ([]conditions.Observation, error) // declaring nothing: retiring its data on that would be acting on an unreadable result. continue } - declared, held := declaredOn(plan) + declared, holder := declaredOn(plan) + // **A holder is asked only once its machine has been sent it and had time to report** (novox/hq + // issue 275): assigned and not pushed yet, it is not there to answer, and "did not say what it + // measured" about it was a warning for a push nobody had made yet. + held := holder != "" && delivered[n.Name].settled(holder, now) machines = append(machines, &machine{name: n.Name, declared: declared, held: held}) } // Every holder asked at once, as D8 asks every ban list. diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go index 2ed87e2..25a3d9f 100644 --- a/cmd/mesh-controller/data_test.go +++ b/cmd/mesh-controller/data_test.go @@ -299,6 +299,10 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent( if _, err := assign(ctx, open, "laptop", "house"); err != nil { t.Fatal(err) } + // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). + for _, node := range []string{"laptop", "anchor"} { + pushedAndApplied(t, open, node) + } conn := onATestBus(t) js, err := broker.Dial(os.Getenv("MESH_TEST_NATS")) diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 1394242..523b248 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -75,7 +75,8 @@ type probe struct { // probe added to a design is a row added here. var probeRegistry = []probe{ {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation", - From: "issues 236, 263", Kind: "declaration-refused", Phase: 1, run: probeDeclarations}, + From: "issues 236, 263, 275", Kind: "declaration-refused", Raises: []string{kindAwaitingPush}, Phase: 1, + run: probeDeclarations}, {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers}, {ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every " + diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index b1949c8..de31ed0 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -378,13 +378,19 @@ func declarationFor(ctx context.Context, open *stores, node string, // So the mesh chooses a port when it commits to sending one, and every other caller reads what // was chosen. A module with nothing assigned yet has never been sent, which is exactly what a // machine "waiting" means — the read needs no number to be right about that. -type Choosing bool +type Choosing int const ( - // Allocating is the send path: what is not assigned yet is assigned now and kept. - Allocating Choosing = true // Reading is every question: what is assigned is used, and nothing is created. - Reading Choosing = false + Reading Choosing = iota + // Allocating is the send path: what is not assigned yet is assigned now and kept. + Allocating + // Foreseeing is Reading, asked ahead of a send (the self-check's D1, novox/hq issue 275): nothing + // is created, and an own secret the next send WOULD make is composed with a stand-in and named + // (sendable.foreseen) rather than failing the composition — while one the send would be refused + // (a bus credential nobody issued, a given secret under an old key) is refused here as it would + // be there. Never sent: the stand-in is not a sealed value. + Foreseeing ) func declarationWith(ctx context.Context, open *stores, node string, @@ -406,7 +412,7 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound} + unbound: with.Unbound, foreseen: composed.Foreseen} // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -507,6 +513,30 @@ func reportLeftOut(node string, declared sendable) { } } +// busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued. +// +// **The broker credential is never invented here** (novox/hq issue 203). Every other own secret is +// the mesh's to make — a password nobody else knows — but this one is an account on the bus, minted +// by `module issue` and sealed by it; a push that made a random one would deliver a file the process +// cannot read and report the machine applied. Refused by name, with the verb — on the send, and on +// a question asked ahead of it (Foreseeing), so that one is never told the push will make it. +func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error { + if name != "broker" { + return nil + } + user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() + if _, minted, err := inv.BusUserHash(ctx, user); err != nil { + return err + } else if !minted { + return fmt.Errorf( + "%s on %s has no bus credential: nothing was issued for %s, and a push "+ + "would seal a placeholder its process cannot read (novox/hq issue 203). "+ + "`module issue %s --node %s`, then push again", + module, node, user, module, node) + } + return nil +} + // renderingFor is everything a node's declaration is composed with, and the node's record. func renderingFor(ctx context.Context, open *stores, node string, plan catalogue.Resolution, settings catalogue.SettingsBy, @@ -524,7 +554,7 @@ func renderingFor(ctx context.Context, open *stores, node string, // consumer is told are all derived from it. // What this machine was already given, for a composition that may not allocate. already := map[string]map[int]int{} - if choosing == Reading { + if choosing != Allocating { held, err := inv.PortsFor(ctx, node) if err != nil { return catalogue.Rendering{}, inventory.Node{}, err @@ -610,6 +640,7 @@ func renderingFor(ctx context.Context, open *stores, node string, // And each module's own secrets — a superuser password, an administrator, an account. Made // per node, so a module running on three machines has three. needed := map[string]map[string]string{} + foreseen := map[string]map[string]bool{} for _, m := range plan.Modules { for name := range m.OwnSecrets { // Minted on the send path and only read on every other. Making one is an insert, and @@ -617,27 +648,29 @@ func renderingFor(ctx context.Context, open *stores, node string, var sealed string var err error if choosing == Allocating { - // **The broker credential is never invented here** (novox/hq issue 203). Every other - // own secret is the mesh's to make — a password nobody else knows — but this one - // is an account on the bus, minted by `module issue` and sealed by it; a push that - // made a random one would deliver a file the process cannot read and report the - // machine applied. Refused by name, with the verb. - if name == "broker" { - user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username() - if _, minted, err := inv.BusUserHash(ctx, user); err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } else if !minted { - return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf( - "%s on %s has no bus credential: nothing was issued for %s, and a push "+ - "would seal a placeholder its process cannot read (novox/hq issue 203). "+ - "`module issue %s --node %s`, then push again", - m.Module, node, user, m.Module, node) - } + if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil { + return catalogue.Rendering{}, inventory.Node{}, err } sealed, err = inv.SecretForModule(ctx, node, m.Module, name) } else { var held bool sealed, held, err = inv.ModuleSecretIfIssued(ctx, node, m.Module, name) + if err == nil && !held && choosing == Foreseeing { + // Asked ahead of the send: what the send would do about it, by the send's own + // rules. Made by it — a stand-in, named; refused by it — refused here, the same + // words (novox/hq issue 275). + if err := busCredentialIssued(ctx, inv, node, m.Module, name); err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if err := inv.WouldMakeSecretForModule(ctx, node, m.Module, name); err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if foreseen[m.Module] == nil { + foreseen[m.Module] = map[string]bool{} + } + foreseen[m.Module][name] = true + continue + } if err == nil && !held { // Never issued, so this machine cannot be running it. Left out rather than // invented: an empty string here would compose a declaration that differs @@ -829,7 +862,7 @@ func renderingFor(ctx context.Context, open *stores, node string, } return catalogue.Rendering{ BusMembership: memberships[node], - Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, + Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Zones: zones, Holders: replicas, Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation, diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index 6476acf..3ccb06e 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -23,6 +23,7 @@ import ( "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/overlay" @@ -32,8 +33,25 @@ import ( // wrong now as observations, or an error when it could not tell — never "nothing" for "could not // look" (ADR 0227 rule 4). +// awaitingPushBound is how long a machine may have something only its next push makes — an own +// secret of a module assigned since its last push — before that is said (novox/hq issue 275). +var awaitingPushBound = 30 * time.Minute + +// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound. +const kindAwaitingPush = "awaiting-push" + // probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator // (mesh-host's `validate`, the package the host runs) takes it. +// +// **Composed as the next push would compose it, without making anything** (Foreseeing, novox/hq issue +// 275). Between `assign` and `push` a module's own secrets are not made yet — the push makes them — +// and a composition that only reads them failed, which raised an urgent "nothing can be sent" about a +// machine the next push sent to without a word. So a secret the push WILL make is composed with a +// stand-in and named; one the push would be refused on is refused here, with the push's words. A +// machine whose declaration composes and validates with only such stand-ins is waiting for a push, +// not broken: nothing is said until awaitingPushBound passes from when it began waiting, and then a +// warning (`awaiting-push`) naming what the push will make — never urgent, because nothing is wrong +// that a push does not fix. func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation, error) { open := d.open nodes, err := open.inventory.Nodes(ctx) @@ -59,10 +77,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation if err != nil && !unresolvable(err) { return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) } - var problems []string + var problems, foreseen []string if err == nil && gensErr == nil { var declared sendable - if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil { + if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Foreseeing); err == nil { + foreseen = declared.foreseen // With the order it was last sent, so the validator reads the envelope a machine is sent // — its epoch included (novox/hq to-be 45 §6). var serr error @@ -94,11 +113,30 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s", n.Name, len(problems), problems[0]), Said: strings.Join(problems, "; ")}) + case len(foreseen) > 0: + since, err := open.inventory.AwaitingSince(ctx, n.Name) + if err != nil { + return nil, err // the store, not the machine: the probe could not run + } + if waited := time.Since(since); waited > awaitingPushBound { + out = append(out, awaitingPush(n.Name, foreseen, since, waited)) + } } } return out, nil } +// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has +// made what it names, past awaitingPushBound (novox/hq issue 275). +func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation { + made := strings.Join(foreseen, ", ") + return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush, + Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+ + "`push %s` sends it", node, waited.Round(time.Minute), made, node), + Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)} +} + // probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its // address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl // takes as final (issue 262). @@ -221,6 +259,11 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err return nil, err } heard := heardMachines(d) + delivered, err := readDeliveries(ctx, d.open.inventory) + if err != nil { + return nil, err + } + now := time.Now() expected := map[string]map[string]bool{} // seat → machine add := func(seat, node string) { if expected[seat] == nil { @@ -248,7 +291,10 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err continue } for _, node := range e.On { - if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) { + // Assigned is not there yet: a holder is expected once its machine was sent it and + // had time to report, never between `assign` and `push` (novox/hq issue 275). + if heard[node] && (s.Scope == catalogue.ScopeNode || !onRecord) && + delivered[node].settled(e.Manifest.Module, now) { add(s.Name, node) } } @@ -277,6 +323,58 @@ func probeHolders(ctx context.Context, d *doctor) ([]conditions.Observation, err return sortedFound(out), nil } +// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report +// before what the declaration carries is expected to answer (novox/hq issue 275). +var reportGrace = 10 * time.Minute + +// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send +// carried, when, and whether the machine has reported acting on it. +type lastSend struct { + // carried is the modules its last send carried; nil when that was not kept (a send from before + // it was, or one by hand), and then every module is taken as carried — as before this existed. + carried map[string]string + // sent is when its current declaration went to it; nil if nothing ever did. + sent *time.Time + // current says its last report names the declaration last sent. + current bool +} + +// settled says whether a module's holder on this machine can be asked to answer now: the machine was +// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to. +// A machine never sent anything holds nothing the mesh put there. +func (d lastSend) settled(module string, now time.Time) bool { + if d.sent == nil { + return false + } + if d.carried != nil { + if _, in := d.carried[module]; !in { + return false + } + } + return d.current || now.Sub(*d.sent) > reportGrace +} + +// readDeliveries is every machine's last send, by name, from the records a send and a report keep. +func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) { + reports, err := inv.LastReports(ctx) + if err != nil { + return nil, err + } + out := map[string]lastSend{} + for _, r := range reports { + builds, known, err := inv.SentBuilds(ctx, r.Node) + if err != nil { + return nil, err + } + d := lastSend{sent: r.Sent, current: r.Current} + if known { + d.carried = builds + } + out[r.Node] = d + } + return out, nil +} + // heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw. func heardMachines(d *doctor) map[string]bool { out := map[string]bool{} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index f0dddbb..eaadeb0 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -54,6 +54,10 @@ type sendable struct { // unbound is every consumer whose credential from this machine is on record and that is bound // elsewhere (novox/hq issue 274); for push and plan to say, never on the wire. unbound []catalogue.Unbound + // foreseen is every own secret composed with a stand-in, as `module/name`: what the next send will + // make (Foreseeing, novox/hq issue 275). Never on the wire, and a declaration that has any is never + // sent. + foreseen []string // Builds is the build of each module this declaration carries — module to the commit its build // was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221). // Composed only on the send path; nil records that it is not known. diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 58141c9..7a2774c 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -87,6 +87,12 @@ type Rendering struct { // Needed is each module's own secrets, sealed to this node, keyed by module and then by the // name the module gave it. Needed map[string]map[string]string + // Foreseen is each own secret not made yet that the next send WILL make, keyed like Needed: a + // composition asked ahead of the send (the self-check's D1) composes it with ForeseenSealed in + // its place and lists it in Composed.Foreseen, rather than failing for a value only a send + // makes. Nil on every composition that is sent, and on every other question, which then refuse + // a secret not made with a *NotMadeError. + Foreseen map[string]map[string]bool // Mesh is every node's address on the private network, which is what a rule saying "from the // mesh" resolves to. Passed in for the same reason grants are: who else is on the network is @@ -275,6 +281,25 @@ type Composed struct { // compose. Its held things are kept and its containers untouched — the machine is told so — // and it is told everything else. LeftOut map[string]string + // Foreseen is every own secret composed with ForeseenSealed in its place (Rendering.Foreseen), + // as `module/name`, sorted: what the next send will make. Never set on a declaration that is + // sent — a placeholder in a sealed file is a credential the process cannot read. + Foreseen []string +} + +// ForeseenSealed is what stands for an own secret a send will make, in a composition asked ahead of +// the send. Not a sealed value: nothing composed with it may be sent. +const ForeseenSealed = "foreseen: made by the next send" + +// NotMadeError is a module's own secret the composition was given no value for (novox/hq issue 275): +// typed, so that a caller can tell "a send would make this, and none has yet" from a composition that +// fails for any other reason without reading the words. +type NotMadeError struct { + Module, Name string +} + +func (e *NotMadeError) Error() string { + return fmt.Sprintf("%s needs a secret called %q and none was made for it", e.Module, e.Name) } // LeftOut is which of this machine's modules a declaration composed with these settings leaves @@ -295,10 +320,12 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { owner := map[string]string{} received := map[string]map[string][]Contribution{} leftOut := map[string]string{} - resources, err := r.compose(with, owner, received, leftOut) + var foreseen []string + resources, err := r.compose(with, owner, received, leftOut, &foreseen) if err != nil { return Composed{}, err } + sort.Strings(foreseen) if with.BusMembership != "" { // The machine's own, not any module's: how it reaches the mesh from now on. Sealed like a // secret and placed where the host looks for exactly this (design 28, task 5.2). @@ -307,7 +334,8 @@ func (r Resolution) Compose(with Rendering) (Composed, error) { "sealed": with.BusMembership, "mode": "0600", }) } - return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut}, nil + return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut, + Foreseen: foreseen}, nil } // BusMembershipID names the resource carrying a machine's membership for the new bus, and @@ -317,7 +345,8 @@ func BusMembershipID() string { return "bus-membership" } const BusMembershipPath = "/var/lib/mesh/membership-next.json" func (r Resolution) compose(with Rendering, owner map[string]string, - received map[string]map[string][]Contribution, leftOut map[string]string) ([]map[string]any, error) { + received map[string]map[string][]Contribution, leftOut map[string]string, + foreseen *[]string) ([]map[string]any, error) { // **A setting is judged where it is stored, and an impossible one costs a module, not a // machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes // this module uncomposable; it is left out of the declaration — its held things kept, its @@ -534,12 +563,17 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } for _, name := range sortedKeys(m.OwnSecrets) { sealed := with.Needed[m.Module][name] + if sealed == "" && with.Foreseen[m.Module][name] { + // Not made, and the next send makes it: composed with a stand-in so that whatever + // else this composition would refuse is still found (novox/hq issue 275). + sealed = ForeseenSealed + *foreseen = append(*foreseen, m.Module+"/"+name) + } if sealed == "" { // Declared and not made. Refused rather than skipped: a module whose own // credential is silently absent starts, fails to authenticate, and the reason is // three layers away from the machine reporting it. - return nil, fmt.Errorf( - "%s needs a secret called %q and none was made for it", m.Module, name) + return nil, &NotMadeError{Module: m.Module, Name: name} } // The runtime's credential belongs to the account the runtime runs as (novox/hq ADR 0175, // to-be 38 WP3): its process is composed `user: ` where the node has one, and a diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 11e8328..0e1dbc6 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -976,6 +976,25 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu return err } +// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue +// 275): the oldest assignment on it made after it was last sent, or — when nothing was assigned since — +// when it was last sent, or when it joined if it never was. The moment a bound on "not pushed yet" is +// read from: every change a push carries happened after the last push, and an assignment is the one +// that says when. +func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) { + var since time.Time + err := i.store.Pool().QueryRow(ctx, + `select coalesce( + (select min(a.assigned) from assignment a + where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)), + n.sent_at, n.created) + from node n where n.name = $1`, name).Scan(&since) + if errors.Is(err, pgx.ErrNoRows) { + return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + return since, err +} + // SentBuilds is the build of each module a machine was last sent, by its name: module to the commit // its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine // last sent before it was, sent a declaration by hand, or one the mesh does not know. diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index d1b83ce..cebe1df 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -327,9 +327,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri return "", err } if key == "" { - return "", fmt.Errorf( - "%s needs a secret and %s has no sealing key, so nothing can be sealed to it", - module, node) + return "", noSealingKey(module, node) } record, err := i.NodeByName(ctx, node) if err != nil { @@ -349,10 +347,7 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri // would put 32 random bytes where a working credential was: the machine would apply it, // report success, and whatever reads it would fail to authenticate somewhere else // entirely — with the mesh insisting the secret was delivered, which it was. - return "", fmt.Errorf( - "%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ - "since generated a new sealing key. The mesh cannot make another; issue it again", - module, node, name, node) + return "", acceptedUnderAnOldKey(module, node, name) } operator, err := i.OperatorKey(ctx) @@ -381,6 +376,51 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri return made.ForConsumer, nil } +// WouldMakeSecretForModule says what SecretForModule would do about an own secret, without doing it: +// nil when it would make one (or one is held), and otherwise the very refusal it would meet. The read +// a question asked ahead of a send uses (novox/hq issue 275), so that "the next push makes this" is +// told apart from "the next push fails on this" by the same rules the push applies. +func (i *Inventory) WouldMakeSecretForModule(ctx context.Context, node, module, name string) error { + key, err := i.SealingKeyOf(ctx, node) + if err != nil { + return err + } + if key == "" { + return noSealingKey(module, node) + } + record, err := i.NodeByName(ctx, node) + if err != nil { + return err + } + var against, origin string + err = i.store.Pool().QueryRow(ctx, + `select node_key, origin from module_secret where node = $1 and module = $2 and name = $3`, + record.ID, module, name).Scan(&against, &origin) + switch { + case errors.Is(err, pgx.ErrNoRows): + return nil + case err != nil: + return err + case against != key && origin == "accepted": + return acceptedUnderAnOldKey(module, node, name) + } + return nil +} + +// noSealingKey is the refusal for a secret on a machine that has no key to seal it to. +func noSealingKey(module, node string) error { + return fmt.Errorf("%s needs a secret and %s has no sealing key, so nothing can be sealed to it", + module, node) +} + +// acceptedUnderAnOldKey is the refusal for a given secret sealed to a key the machine no longer has. +func acceptedUnderAnOldKey(module, node, name string) error { + return fmt.Errorf( + "%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ + "since generated a new sealing key. The mesh cannot make another; issue it again", + module, node, name, node) +} + // AcceptSecretForModule keeps a value somebody supplied as a module's own secret. // // The counterpart to SecretForModule, which generates one. Some of what a module needs the mesh