From 4bff67ec69b66a7fc327c8bdf73d84d62f279e83 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 29 Aug 2026 20:32:18 +0200 Subject: [PATCH] A machine waiting to be enrolled is not a broken one The launcher already ran `host run`, and `run` on a machine with no identity exited with an error. So a freshly installed host, sitting exactly as intended waiting for somebody to bring it a token, would have counted three failed starts and rolled back its own installation. It waits now, and says what it is waiting for. That is the *hosted* state from the lifecycle: the host is running, it has no identity, and there is nobody to link to. Every machine passes through it. An identity that exists and cannot be read is still a fault rather than a wait. Treating that as "not enrolled yet" would leave a node sitting quietly for ever while the mesh believes it is a member. Also: a node now says it is there once a minute. Nothing but its name, because anything more would be a report, and reports are rare where this is constant -- reading one as the other would make a quiet node look like a stale one. Not published mandatory, unlike a report: losing one is nothing, the next is a minute away, and the mesh reads a gap rather than counting arrivals. Verified in the lab: a node was stopped and the mesh said "out of touch 4m", then it was started and the mesh said "here" again, without anything else being touched. --- cmd/mesh-host/main.go | 51 +++++++++++++++++++++++++++--- examples/substrate-first-node.lock | 4 +-- internal/link/messages.go | 14 ++++++++ internal/link/run.go | 34 ++++++++++++++++++++ 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 38249bb..1aa17cb 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -535,14 +535,20 @@ func firstNonEmpty2(values ...[]byte) []byte { // is down it is disconnected, which is an ordinary situation rather than a failure — the machine // keeps running whatever it was last told, from its own store. func runLink(ctx context.Context, opts options) error { - mine, err := identity.Load(identity.Path(opts.state)) - if errors.Is(err, identity.ErrNoIdentity) { - return errors.New("this machine has not joined a mesh. Enrol it first: " + - "mesh-host enrol --token --name ") - } + // A machine that has not enrolled waits here rather than failing. It is *hosted*: the host is + // running, it has no identity, and there is nobody to link to — an ordinary state, and the + // one every machine passes through (novox/hq 09-the-node-lifecycle). + // + // Exiting instead would be worse than untidy. The launcher counts a failed start, and three + // of them roll the binary back — so a freshly installed host, waiting to be enrolled exactly + // as intended, would undo its own installation. + mine, err := waitForEnrolment(ctx, opts) if err != nil { return err } + if mine.Node == "" { + return nil // asked to stop while waiting + } fmt.Printf("node %s, linking to %s\n", mine.Node, mine.Membership.Broker) @@ -668,3 +674,38 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D } return report } + +// waitForEnrolment returns this node's identity, waiting for one if it has none. +// +// It applies the carried bundle first, if there is one, because that is what a first node does +// before there is a mesh at all — and a machine that has been installed and not yet enrolled +// should still be whatever its bundle says it is. +func waitForEnrolment(ctx context.Context, opts options) (identity.Identity, error) { + const look = 5 * time.Second + + said := false + for { + mine, err := identity.Load(identity.Path(opts.state)) + if err == nil { + return mine, nil + } + if !errors.Is(err, identity.ErrNoIdentity) { + // An identity that exists and cannot be read is a fault, not a wait. Treating it as + // "not enrolled yet" would leave a node sitting quietly for ever while the mesh + // believes it is a member. + return identity.Identity{}, err + } + + if !said { + fmt.Println("this machine has not joined a mesh, and is waiting to be told which one.") + fmt.Println(" enrol it with: mesh-host enrol --token --name ") + said = true + } + + select { + case <-ctx.Done(): + return identity.Identity{}, nil + case <-time.After(look): + } + } +} diff --git a/examples/substrate-first-node.lock b/examples/substrate-first-node.lock index f180a52..3b1b300 100644 --- a/examples/substrate-first-node.lock +++ b/examples/substrate-first-node.lock @@ -79,7 +79,7 @@ "command": ["docker", "run", "--rm", "--network", "container:mesh-store", "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", - "192.0.2.250:5000/mesh-control@sha256:1900893c8d175f1f955569dbead167e58ec3a593fc2ad9170fd936e0a8a55c3a", + "192.0.2.250:5000/mesh-control@sha256:5aaaea5f3d0daa7a4cdde4e13dbddc61176d46487e6a89e6d992808e5e771593", "migrate"], "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key"] }, @@ -113,7 +113,7 @@ "id": "control-plane", "type": "container", "name": "mesh-control", - "image": "192.0.2.250:5000/mesh-control@sha256:1900893c8d175f1f955569dbead167e58ec3a593fc2ad9170fd936e0a8a55c3a", + "image": "192.0.2.250:5000/mesh-control@sha256:5aaaea5f3d0daa7a4cdde4e13dbddc61176d46487e6a89e6d992808e5e771593", "network": "host", "args": ["serve"], "volumes": ["mesh-broker-tls:/broker-tls:ro"], diff --git a/internal/link/messages.go b/internal/link/messages.go index 46e20c2..f82489c 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -8,8 +8,22 @@ package link // queue, so it can say these things and nothing else. const ( KeyReport = "report" + KeyAlive = "alive" ) +// Alive is a node saying nothing except that it is there. +// +// novox/hq 09-the-node-lifecycle: *how long it has been disconnected is a fact the mesh must +// hold, and nothing holds it today. Without it, a node running last month's assignments looks +// exactly like one that is current.* +// +// Separate from a report because the two happen at completely different rates: a node is alive +// constantly and applies something rarely, and reading one as the other would make a quiet node +// look like a stale one. +type Alive struct { + Node string `json:"node"` +} + // Signed is a declaration and the signature over it. // // novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is diff --git a/internal/link/run.go b/internal/link/run.go index 882e8b6..091e2e2 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -19,6 +19,13 @@ import ( // the first means somebody is trying, the second means something is broken. var ErrForged = errors.New("this declaration was not signed by the mesh this node joined") +// AliveEvery is how often a node says it is there. +// +// Often enough that "no word for five minutes" means something, rarely enough that a hundred +// nodes are not a hundred messages a second. The mesh reads absence rather than presence, so what +// matters is the interval being known and steady. +const AliveEvery = 60 * time.Second + // Membership is what a node needs to reach its mesh again, held by the caller. type Membership struct { Node string @@ -153,6 +160,13 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout // reads as still broken when it means the opposite. say("in the mesh, consuming " + queue) + // A word every so often, so the mesh can tell a node that is quiet from one that is gone. + // Cheap on purpose: it carries a name and nothing else, because anything more would be a + // report, and reports are rare where this is constant. + beat := time.NewTicker(AliveEvery) + defer beat.Stop() + publishAlive(ctx, channel, m, say, timeout) + closed := conn.NotifyClose(make(chan *amqp.Error, 1)) // Published mandatory, so the broker hands back anything it cannot route rather than @@ -171,6 +185,8 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout select { case <-ctx.Done(): return nil + case <-beat.C: + publishAlive(ctx, channel, m, say, timeout) case reason := <-closed: return fmt.Errorf("the link closed: %v", reason) case delivery, ok := <-deliveries: @@ -236,3 +252,21 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) } } + +// publishAlive says this node is here, and nothing else. +func publishAlive(ctx context.Context, channel *amqp.Channel, m Membership, say Announce, + timeout time.Duration) { + body, err := json.Marshal(Alive{Node: m.Node}) + if err != nil { + return + } + publish, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + // Not mandatory, unlike a report. Losing one is nothing: the next is a minute away, and the + // mesh is reading a gap rather than counting arrivals. Insisting on delivery would turn a + // harmless miss into a logged failure every minute. + if err := channel.PublishWithContext(publish, Exchange, KeyAlive, false, false, + amqp.Publishing{ContentType: "application/json", Body: body}); err != nil { + say("could not tell the mesh this node is here: " + err.Error()) + } +}