novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Witness the node tools alone: the controller's lease names no build any more

Browse Source 
The controller ended by its unassignment (issue 1126, ADR 0590), so no build is judged by the lease: the lease
witness, its contract and the direct get the witness asked it with are removed, and its tests now prove each rollback
path on the node tools' PING.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 11:22:21 +01:00
parent 84737f4cb2
commit ff4dc71ce4
11 changed files with 89 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/link/messages.go
+2 -3
View File
@@ -488,9 +488,8 @@ const WitnessContract = 1
// The core components a witness judges (to-be 45 §8), as a rollback names them.
const (
ComponentEngine = "node-engine"
ComponentController = "controller"
ComponentNodeTools = "node-tools"
ComponentEngine = "node-engine"
ComponentNodeTools = "node-tools"
)
// What a witness concluded, as a rollback says it.
internal/link/messages_test.go
+1 -1
View File
@@ -27,7 +27,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
// witness contract this host keeps.
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract},
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
[]string{"node", "rollbacks", "witness"}},
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
[]string{"component", "from", "to", "outcome", "why", "at"}},
internal/link/testdata/witness-grants.conf
Vendored
+3 -5
View File
@@ -3,16 +3,15 @@
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
#
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool
# runtime, `admin` for the controller writing the lease.
# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
jetstream: enabled
accounts {
MESH {
jetstream: enabled
users = [
{ user: "node.anchor", password: "a", permissions: {
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] }
publish: { allow: ["$SRV.PING.node-tools.anchor"] }
subscribe: { allow: ["_INBOX.node.anchor.>"] }
} }
{ user: "node.bare", password: "b", permissions: {
@@ -23,7 +22,6 @@ accounts {
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "admin", password: "x" }
]
}
}
internal/link/witnessing.go
+4 -46
View File
@@ -10,14 +10,11 @@ import (
"github.com/nats-io/nats.go"
)
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
// node tools it placed on this machine, by what the bus says about them rather than by what they
// say about themselves.
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
// machine, by what the bus says about them rather than by what they say about themselves.
//
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
// by JetStream's direct get — one subject, no stream information, nothing written; and this
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
// so only this machine's runtime can answer it.
// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
// PING by its name and this machine's, so only this machine's runtime can answer it.
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
type Asker interface {
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
Ping(ctx context.Context, service, id string) error
}
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
func DirectGetSubject(bucket, key string) string {
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
}
// PingSubject is the services protocol's PING of one instance of one service.
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
return nil
}
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
subject := DirectGetSubject(bucket, key)
before := l.conn.LastError()
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
switch {
case errors.Is(err, nats.ErrNoResponders):
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
case err != nil:
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
}
if msg.Header != nil {
switch status := msg.Header.Get("Status"); status {
case "":
case "404":
return nil, false, nil
default:
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
bucket, status, msg.Header.Get("Description"))
}
switch msg.Header.Get("KV-Operation") {
case "DEL", "PURGE":
return nil, false, nil
}
}
if len(msg.Data) == 0 {
return nil, false, nil
}
return msg.Data, true, nil
}
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
subject := PingSubject(service, id)
before := l.conn.LastError()
internal/link/witnessing_nats_test.go
+2 -62
View File
@@ -2,7 +2,6 @@ package link
import (
"context"
"encoding/json"
"errors"
"os"
"testing"
@@ -12,7 +11,7 @@ import (
)
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
// whether the direct get and the PING work as the two subjects named, and whether a host without them
// whether the PING works as the subject named, and whether a host without them
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_GRANTS")
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Cleanup(conn.Close)
return conn
}
admin := dial("admin", "x", "_INBOX.admin")
js, err := admin.JetStream()
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue("mesh-controller_lease")
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
if err != nil {
t.Fatal(err)
}
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
ask := func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 2*time.Second)
}
// Nobody holds it.
ctx, cancel := ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("an empty lease read as %v, %v", found, err)
}
cancel()
// Held.
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
cancel()
if err != nil || !found {
t.Fatalf("a held lease read as %v, %v", found, err)
}
var holder struct {
Instance string `json:"instance"`
}
if json.Unmarshal(value, &holder); holder.Instance == "" {
t.Fatalf("the lease's value is %s", value)
}
// Given back.
if err := kv.Delete("holder"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("a lease given back read as %v, %v", found, err)
}
cancel()
// A host without the grant cannot ask — which is not "nobody holds it".
ctx, cancel = ask()
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
}
cancel()
// No runtime on the bus: not there to answer.
ctx, cancel = ask()
ctx, cancel := ask()
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
t.Fatalf("a PING nobody serves is %v, want no answer", err)
}
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
}
cancel()
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
}
cancel()
}