Merge pull request 'A first node gets as far as its own bus: three faults on the way' (#50) from fix/one-foundation-on-the-bus-the-mesh-runs-on into main

novox/hq 04-ISSUES/146. Each was right while the mesh ran on the previous
broker, and nothing has raised a foundation since it changed.

The bus's certificate is made by the program that needs it rather than by
openssl inside the broker's image — the bus's image is Alpine with a shell and
no openssl, so the step exited 127 and no mesh could be raised. Self-signed as
before and on purpose; --user 0:0 because the volume is root's and the control
plane's image runs as nobody.

Enrolment no longer opens a raw TLS connection to check the pin: NATS speaks
its own protocol and upgrades afterwards, so the handshake met a plaintext
greeting. The client that presents the token carries the same pinned config
and verifies inside its own handshake, so the secret still leaves only after
the certificate is checked. The raw dial stays as what its tests prove, and is
no longer a path anything takes.

And the token says which bus it is for. Empty meant 'whatever the mesh runs
today' while two buses existed and became a refusal the moment one did.

It now stops at the bus's user list, which is the genesis half of 146.

cmd/mesh-host/main.go

@@ -706,15 +706,18 @@ func enrol(ctx context.Context, opts options) error {
 	fmt.Printf("  signing key         %s\n",
 		base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
 
-	// The check that has to happen before this machine says anything.
+	// **The pin is checked by the connection that presents this token, not by a dial of our own**
-	conn, err := link.Dial(token.Broker, token.Fingerprint, opts.timeout)
+	// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
-	if err != nil {
+	// against the broker the mesh used to run and cannot work against the one it runs now: NATS
-		return err
+	// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
-	}
+	// with a plaintext line and the enrolment failed with "first record does not look like a TLS
-	defer conn.Close()
+	// handshake" — on every node that has tried to join since the bus changed, which is why this
-	fmt.Println("\nthe broker presented the certificate this token pins")
+	// went unnoticed: none had.
-
+	//
-	conn.Close()
+	// What ADR 0004 requires still holds, and holds better: the client that presents the token
+	// carries the same pinned configuration, reads the server's greeting, upgrades, and the
+	// verification runs inside that handshake — so the one-time secret is sent only after the
+	// certificate has been checked, and nothing of this node's reaches an impostor.
 
 	mine, err := identity.Generate(*name)
 	if err != nil {
@@ -788,10 +791,16 @@ func enrol(ctx context.Context, opts options) error {
 	proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
 		sealing.Public, serving.Public))
 	// The token says where to go and which certificate that address must present. It says nothing
-	// about which bus is there, and does not need to: every token names the one the mesh runs on
+	// about which bus is there, and does not need to: there is one, and this host knows which
-	// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
+	// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
+	//
+	// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
+	// while two buses existed and became a refusal the moment one did: an empty transport is not
+	// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
+	// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
+	// changed.
 	reply, err := link.Enrol(ctx,
-		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
+		link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
 		*name, token.Secret,
 		mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
 		opts.timeout)
@@ -1246,6 +1255,21 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
 	return applyAndKeep(ctx, opts, raw, nil, sched, say)
 }
 
+// announceOr is what the apply writes its detail with, given what the caller has to say things with.
+//
+// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
+// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
+// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
+//
+// Named rather than written inline at the call site so both paths reach the apply the same way, and so
+// a reader asking "where does the apply's output go" finds one answer.
+func announceOr(say link.Announce) func(string) {
+	if say == nil {
+		return func(string) {}
+	}
+	return say
+}
+
 // applying serialises applies within this process.
 //
 // **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
@@ -1308,8 +1332,18 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
 
 	// Declared, not carried. A declaration from the mesh removes only what the mesh previously
 	// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
+	// **What the apply says goes to the console, which is the journal when this runs as a service.**
+	//
+	// It was nil, and nil is silence. The one-shot path has always passed a real one, so every detail
+	// the apply produces — a file held, a container replaced, the found firewall retired — was visible
+	// when a person ran it by hand and discarded in the way the host actually runs. Measured: after a
+	// machine was converged and its found firewall was not retired, what the host decided was
+	// unrecoverable, because it had said it to nobody (novox/hq 04-ISSUES/143).
+	//
+	// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
+	// no new mechanism — only for the argument to be passed.
 	outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
-		apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
+		apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
 
 	// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
 	// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).

cmd/mesh-host/main_test.go

@@ -568,3 +568,28 @@ func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
 		t.Fatal("a refused report is offered as news about the machine")
 	}
 }
+
+// **A host running as a service says what its apply did.**
+//
+// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
+// has always passed a real function, so everything the apply says was visible when a person ran it by
+// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
+// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
+// had been said to nobody (novox/hq 04-ISSUES/143).
+//
+// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
+// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
+// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
+// carries the apply's detail, which is how this fix was verified.
+func TestTheApplysLogIsNeverNil(t *testing.T) {
+	if announceOr(nil) == nil {
+		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
+	}
+	announceOr(nil)("this goes nowhere and must not panic")
+
+	var said []string
+	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
+	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
+		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
+	}
+}

examples/foundation-first-node-nats.lock

@@ -127,12 +127,21 @@
     {
       "id": "bus-certificate",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
-                  "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
+      // shell and no openssl, and no other image the bundle names has one either. So the program
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
+      // that needs the certificate writes it — already on this machine, since the schema step ran
-                 "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
     },
     {
       "id": "bus-conf-dir",

internal/link/pinned.go

@@ -73,8 +73,19 @@ func PinnedConfig(pin string) (*tls.Config, error) {
 	}, nil
 }
 
-// Dial opens a TLS connection to the broker, refusing anything but the pinned certificate.
+// dialPinned completes a TLS handshake against an address, refusing anything but the pinned
-func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
+// certificate.
+//
+// **Not how the bus is reached, and it used to be** (novox/hq 04-ISSUES/146). Enrolment opened one
+// of these before it said anything, which was right while the broker answered TLS immediately and
+// wrong the moment the mesh moved to a bus that speaks its own protocol first. The pin itself was
+// never the problem — PinnedConfig is what the NATS client is given, and the verification runs
+// inside the handshake that client performs.
+//
+// It stays here because this is where the pin is proven: the tests beside it run a real TLS server
+// and assert that a wrong certificate is refused before a byte of application data is sent. What it
+// must not become again is something a caller uses to reach the bus.
+func dialPinned(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 	config, err := PinnedConfig(pin)
 	if err != nil {
 		return nil, err
@@ -86,7 +97,7 @@ func Dial(address, pin string, timeout time.Duration) (*tls.Conn, error) {
 		if errors.Is(err, ErrWrongCertificate) {
 			return nil, err
 		}
-		return nil, fmt.Errorf("cannot reach the broker at %s: %w", address, err)
+		return nil, fmt.Errorf("cannot reach %s: %w", address, err)
 	}
 	return conn, nil
 }

internal/link/pinned_test.go

@@ -63,7 +63,7 @@ func server(t *testing.T) (address string, fingerprint string) {
 
 func TestTheRightBrokerIsAccepted(t *testing.T) {
 	address, pin := server(t)
-	conn, err := Dial(address, pin, 5*time.Second)
+	conn, err := dialPinned(address, pin, 5*time.Second)
 	if err != nil {
 		t.Fatalf("the broker its token describes was refused: %v", err)
 	}
@@ -76,7 +76,7 @@ func TestADifferentBrokerIsRefused(t *testing.T) {
 	address, _ := server(t)
 	_, other := server(t)
 
-	_, err := Dial(address, other, 5*time.Second)
+	_, err := dialPinned(address, other, 5*time.Second)
 	if err == nil {
 		t.Fatal("a broker presenting a different certificate was accepted")
 	}
@@ -130,7 +130,7 @@ func TestNothingIsSentToTheWrongBroker(t *testing.T) {
 
 	// A pin for a certificate this server does not have.
 	_, elsewhere := server(t)
-	if _, err := Dial(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
+	if _, err := dialPinned(listener.Addr().String(), elsewhere, 5*time.Second); err == nil {
 		t.Fatal("the impostor was accepted")
 	}
 	if n := <-received; n > 0 {
@@ -160,7 +160,7 @@ func TestAnUnreachableBrokerIsAnOrdinaryFailure(t *testing.T) {
 	address := listener.Addr().String()
 	listener.Close()
 
-	_, err = Dial(address, pin, 2*time.Second)
+	_, err = dialPinned(address, pin, 2*time.Second)
 	if err == nil {
 		t.Fatal("dialling a closed port succeeded")
 	}