Announce the controller's states, keep every module's word, and grant a read only where its owner announced the state readable; the copied list of the controller's mirrors goes (issue 1790, D1-D4, ADR 0495, ADR 0513 §3)

cmd/mesh-controller/build.go

@@ -683,11 +683,15 @@ func publishable(result link.BuildResult, follows string) error {
 
 // answers is what the three questions came back with, read once.
 type answers struct {
-	wrong   []inventory.Doing
-	nodes   []inventory.Node
-	quiet   []inventory.Node
-	behind  map[string][]string
-	sources map[string]inventory.Source
+	// notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495,
+	// issue 1790): read from `reads` as before until they do; notAnnouncedWhy why that could not be read.
+	notAnnounced    []string
+	notAnnouncedWhy string
+	wrong           []inventory.Doing
+	nodes           []inventory.Node
+	quiet           []inventory.Node
+	behind          map[string][]string
+	sources         map[string]inventory.Source
 	// waiting is every machine not running what the mesh would send it.
 	waiting []inventory.Machine
 	// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR

cmd/mesh-controller/main.go

@@ -54,6 +54,8 @@ func run() error {
 		return fmt.Errorf("no command given")
 	}
 
+	// The bus's users are composed from what each module announced it keeps (novox/hq ADR 0495, issue 1790).
+	inventory.Announcements = readAnnouncements
 	// Every connection this process dials says what it is (novox/hq issue 327).
 	broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
 

cmd/mesh-controller/push.go

@@ -289,6 +289,14 @@ func serve(ctx context.Context) (err error) {
 	// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
 	// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
 	watchNewModules(ctx, open)
+	// And says what it keeps, and keeps what every module says it keeps (novox/hq ADR 0495, issue 1790): a changed word
+	// sends the machine holding the bus its new user list.
+	if err := announceOwnStates(ctx, server.JetStream()); err != nil {
+		fmt.Printf("states: %v\n", err)
+	}
+	if err := keepAnnouncements(ctx, server.JetStream(), open); err != nil {
+		fmt.Printf("states: the states the modules announce are not kept: %v\n", err)
+	}
 	// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
 	// to-be 45 §6).
 	link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })

cmd/mesh-controller/readable.go

@@ -53,6 +53,11 @@ type meshStatus struct {
 	// Per machine, and data. One node failing must never take the document away from a reader
 	// asking about the others.
 	Unresolved []machineUnresolved `json:"unresolved"`
+	// NotAnnounced are the modules others read the state of that have not said what they keep (novox/hq ADR 0495,
+	// issue 1790): their readers are granted from `reads` as before until they do. NotAnnouncedWhy is why it could not
+	// be read.
+	NotAnnounced    []string `json:"not-announced,omitempty"`
+	NotAnnouncedWhy string   `json:"not-announced-why,omitempty"`
 	// Network is why the private network could not be computed, when it could not; absent when it
 	// could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
 	// network, and a mesh whose hub is that node has no hub.
@@ -247,7 +252,8 @@ func statusAsJSON(asked answers) ([]byte, error) {
 	out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
 		Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
 		Reported: []machineReported{}, Unresolved: []machineUnresolved{},
-		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds)}
+		Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds),
+		NotAnnounced: asked.notAnnounced, NotAnnouncedWhy: asked.notAnnouncedWhy}
 	// In a stated order, so two readings of an unchanged mesh are the same document.
 	untakenNodes := make([]string, 0, len(asked.untaken))
 	for name := range asked.untaken {

cmd/mesh-controller/states_announced.go

@@ -0,0 +1,225 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/announce"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// What each module keeps, and who may read it (novox/hq ADR 0495 clauses 2 to 5, ADR 0513 §3; issue 1790, D1-D3).
+//
+// Every module says on the bus which states it keeps and which of them others may read (announce.Subject), from its
+// code; the controller says its own (D1). The serving controller keeps the last word of each module in its bucket
+// (D2), and the bus's users are composed from it: a reader's `reads: <owner>.<name>` is granted only when the owner
+// announced that state readable; an owner that has announced nothing yet is read from `reads` as before (the
+// transition), and `status` names it. A changed word composes the bus's users again: the machine holding the bus is
+// sent its new user list (D3). No timer: the words wake it.
+
+// announceOwnStates says what the controller keeps, on its own event subject (D1).
+func announceOwnStates(ctx context.Context, js *broker.JetStream) error {
+	host, _ := os.Hostname()
+	body, err := announce.Encode(broker.ControllerAnnouncement(host))
+	if err != nil {
+		return err
+	}
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	if _, err := api.Publish(ctx, announce.Subject(broker.ControllerSeat), body); err != nil {
+		return fmt.Errorf("saying what the controller keeps: %w", err)
+	}
+	return nil
+}
+
+// keepAnnouncements follows every module's word on what it keeps (D2) and keeps the last of each, from the last each
+// module said: one module's word in the bucket under its name. A word whose module is not the subject's is refused,
+// said; a changed word composes the bus's users again (D3).
+func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores) error {
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket)
+	if err != nil {
+		return fmt.Errorf("the bucket of the states announced: %w", err)
+	}
+	stream, err := api.Stream(ctx, broker.EventsStream)
+	if err != nil {
+		return fmt.Errorf("the events stream: %w", err)
+	}
+	consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
+		FilterSubjects: []string{announce.SubjectAll}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
+	if err != nil {
+		return fmt.Errorf("following the states announced: %w", err)
+	}
+	consuming, err := consumer.Consume(func(m jetstream.Msg) {
+		changed, err := keepAnnouncement(ctx, kv, m.Subject(), m.Data())
+		if err != nil {
+			fmt.Printf("states: %v\n", err)
+			return
+		}
+		if changed {
+			go composeAnnouncedReads(context.WithoutCancel(ctx), open)
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("following the states announced: %w", err)
+	}
+	go func() {
+		<-ctx.Done()
+		consuming.Stop()
+	}()
+	return nil
+}
+
+// announcedKeeper is the bucket of the states announced, as keepAnnouncement writes it.
+type announcedKeeper interface {
+	Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)
+	Put(ctx context.Context, key string, value []byte) (uint64, error)
+}
+
+// keepAnnouncement keeps one module's word, and says whether it changed what the bucket held.
+func keepAnnouncement(ctx context.Context, kv announcedKeeper, subject string, body []byte) (bool, error) {
+	s, err := announce.Decode(body)
+	if err != nil {
+		return false, fmt.Errorf("a word on %s is not read: %w", subject, err)
+	}
+	if subject != announce.Subject(s.Module) {
+		return false, fmt.Errorf("%s says what %s keeps, and a module says only what it keeps itself: refused", subject,
+			s.Module)
+	}
+	key, err := announce.Key(s.Module)
+	if err != nil {
+		return false, err
+	}
+	stored, err := announce.Encode(s)
+	if err != nil {
+		return false, err
+	}
+	if was, err := kv.Get(ctx, key); err == nil {
+		if string(was.Value()) == string(stored) {
+			return false, nil
+		}
+	} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return false, fmt.Errorf("the last word of %s cannot be read: %w", s.Module, err)
+	}
+	if _, err := kv.Put(ctx, key, stored); err != nil {
+		return false, fmt.Errorf("the word of %s cannot be kept: %w", s.Module, err)
+	}
+	fmt.Printf("states: %s keeps %s\n", s.Module, statesWords(s))
+	return true, nil
+}
+
+// statesWords is an announcement as a person reads it: each state, readable ones marked.
+func statesWords(s announce.States) string {
+	if len(s.States) == 0 {
+		return "nothing"
+	}
+	var out []string
+	for _, k := range s.States {
+		if k.Readable {
+			out = append(out, k.Name+" (readable)")
+		} else {
+			out = append(out, k.Name)
+		}
+	}
+	return strings.Join(out, ", ")
+}
+
+// readAnnouncements is the last word of every module, by module (what inventory.Announcements reads): none when the
+// bucket is not there yet, which reads every owner as not yet announced.
+func readAnnouncements(ctx context.Context) (map[string]announce.States, error) {
+	js, err := aBus()
+	if err != nil {
+		return nil, err
+	}
+	defer js.Close()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		// empty-on-error: not an error; no module announced yet, so every read is granted from `reads` as before
+		return map[string]announce.States{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	keys, err := kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		// empty-on-error: not an error; no module announced yet
+		return map[string]announce.States{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]announce.States{}
+	for _, key := range keys {
+		e, err := kv.Get(ctx, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		s, err := announce.Decode(e.Value())
+		if err != nil {
+			return nil, fmt.Errorf("the kept word of %s is not read: %w", key, err)
+		}
+		out[s.Module] = s
+	}
+	return out, nil
+}
+
+// composeAnnouncedReads sends the machine holding the bus its user list when a changed word changed it (D3): as a new
+// controller's user list is carried (carryUserList), not when that machine is held.
+func composeAnnouncedReads(ctx context.Context, open *stores) {
+	holder, behind, err := brokerBehind(ctx, open, nil)
+	if err != nil {
+		fmt.Printf("states: whether the bus's user list is behind the states announced cannot be read: %v\n", err)
+		return
+	}
+	if holder == "" || !behind {
+		return
+	}
+	held, err := heldMachines(ctx, open, []string{holder})
+	if err != nil {
+		fmt.Printf("states: whether %s may be sent its new user list cannot be read: %v\n", holder, err)
+		return
+	}
+	if why, isHeld := held[holder]; isHeld {
+		fmt.Printf("states: the user list the states announced compose is not sent to %s, which holds the bus: %s — "+
+			"`push %s` sends it\n", holder, strings.Join(why, "; "), holder)
+		return
+	}
+	if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
+		fmt.Printf("states: the user list the states announced compose could not be sent to %s: %v\n", holder, err)
+		return
+	}
+	fmt.Printf("states: sent %s, which holds the bus, the user list the states announced compose\n", holder)
+}
+
+// notYetAnnounced are the owners some module reads the state of and that have announced nothing yet: what `status`
+// names while the transition lasts.
+func notYetAnnounced(ctx context.Context, inv *inventory.Inventory) ([]string, error) {
+	if inventory.Announcements == nil {
+		return nil, nil
+	}
+	records, err := inv.BusRecords(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if records.Announced == nil {
+		return nil, errors.New("the states announced cannot be read")
+	}
+	return broker.NotAnnounced(records), nil
+}

cmd/mesh-controller/states_announced_integration_test.go

@@ -0,0 +1,118 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"slices"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/announce"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// D1-D3 whole against a real bus and the store (ADR 0495 checks §2, §3, §5; ADR 0513 §3; issue 1790): the controller
+// says what it keeps and keeps what every module says; a reader of a state its owner announced readable is granted it,
+// one its owner announced and did not make readable is refused and said, and one of an owner that announced nothing
+// yet is granted as before and named as not yet announced.
+func TestTheStatesAnnouncedGrantTheReads(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if err := broker.AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	servingBus.Store(js)
+	t.Cleanup(func() { servingBus.Store(nil) })
+	was := inventory.Announcements
+	inventory.Announcements = readAnnouncements
+	t.Cleanup(func() { inventory.Announcements = was })
+
+	// The owner keeps two states, one readable; the reader reads both and a state of a module that says nothing.
+	register := func(m catalogue.Manifest) {
+		t.Helper()
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
+			Path: "modules/" + m.Module, BuiltFrom: "c0"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	register(catalogue.Manifest{Module: "owner", Version: "1", State: []catalogue.StateDeclaration{{Name: "open"}, {Name: "closed"}}})
+	register(catalogue.Manifest{Module: "quiet", Version: "1", State: []catalogue.StateDeclaration{{Name: "any"}}})
+	register(catalogue.Manifest{Module: "reader", Version: "1", Reads: []string{"owner.open", "owner.closed", "quiet.any"}})
+	if _, err := inv.AddNode(ctx, "one"); err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range []string{"owner", "quiet", "reader"} {
+		if _, err := inv.Assign(ctx, "one", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	watching, stop := context.WithCancel(context.Background())
+	t.Cleanup(stop)
+	if err := announceOwnStates(ctx, js); err != nil {
+		t.Fatal(err)
+	}
+	if err := keepAnnouncements(watching, js, open); err != nil {
+		t.Fatal(err)
+	}
+	api, _ := jetstream.New(js.Conn())
+	word, _ := announce.Encode(announce.States{Module: "owner", Node: "one",
+		States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}})
+	if _, err := api.Publish(ctx, announce.Subject("owner"), word); err != nil {
+		t.Fatal(err)
+	}
+	becomes(t, 10*time.Second, "both words kept", func() bool {
+		a, err := readAnnouncements(ctx)
+		return err == nil && len(a) == 2 && a["owner"].Readable("open") && a[broker.ControllerSeat].Readable(broker.PlansBuilding)
+	})
+
+	records, err := inv.BusRecords(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var reader broker.Declared
+	for _, d := range records.Assigned["one"] {
+		if d.Module == "reader" {
+			reader = d
+		}
+	}
+	if !slices.Equal(reader.Reads, []string{"owner.open", "quiet.any"}) || !slices.Equal(reader.ReadsRefused, []string{"owner.closed"}) {
+		t.Fatalf("0495 §3: the reader is granted %v and refused %v", reader.Reads, reader.ReadsRefused)
+	}
+	if got := broker.NotAnnounced(records); !slices.Equal(got, []string{"quiet"}) {
+		t.Fatalf("not yet announced: %v", got)
+	}
+	users, err := broker.Users(records)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, u := range users {
+		if u.Kind != broker.KindModule || u.Module != "reader" {
+			continue
+		}
+		perms, err := broker.PermissionsFor(u)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if slices.ContainsFunc(perms.Publish, func(s string) bool { return s == "$JS.API.DIRECT.GET.KV_owner_closed.>" }) {
+			t.Fatal("0495 §3: the reader may read the state its owner did not announce readable")
+		}
+		if !slices.Contains(perms.Publish, "$JS.API.DIRECT.GET.KV_owner_open.>") {
+			t.Fatal("0495 §3: the reader may not read the state its owner announced readable")
+		}
+	}
+}

cmd/mesh-controller/states_announced_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/announce"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// announcedInMemory is the bucket of the states announced, in memory.
+type announcedInMemory map[string][]byte
+
+type announcedEntry struct {
+	jetstream.KeyValueEntry
+	value []byte
+}
+
+func (e announcedEntry) Value() []byte { return e.value }
+
+func (k announcedInMemory) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
+	v, ok := k[key]
+	if !ok {
+		return nil, jetstream.ErrKeyNotFound
+	}
+	return announcedEntry{value: v}, nil
+}
+
+func (k announcedInMemory) Put(_ context.Context, key string, value []byte) (uint64, error) {
+	k[key] = value
+	return uint64(time.Now().UnixNano()), nil
+}
+
+// D2 (ADR 0495 clause 2; issue 1790): a module's word is kept under its name, changed only when it changes, and a word
+// about another module than its subject's is refused.
+func TestAModulesWordIsKeptUnderItsNameAndOnlyItsOwn(t *testing.T) {
+	k := announcedInMemory{}
+	word, _ := announce.Encode(announce.States{Module: "keeper", States: []announce.State{{Name: "things", Readable: true}}})
+	changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word)
+	if err != nil || !changed || string(k["keeper"]) != string(word) {
+		t.Fatalf("the first word: %v %v %q", changed, err, k["keeper"])
+	}
+	if changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word); err != nil || changed {
+		t.Fatalf("the same word again changed the bucket: %v %v", changed, err)
+	}
+	other, _ := announce.Encode(announce.States{Module: "victim", States: []announce.State{{Name: "secrets", Readable: true}}})
+	if _, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), other); err == nil ||
+		!strings.Contains(err.Error(), "only what it keeps itself") {
+		t.Fatalf("a word about another module was kept: %v", err)
+	}
+	if _, found := k["victim"]; found {
+		t.Fatal("a word about another module was kept")
+	}
+}

cmd/mesh-controller/status.go

@@ -85,6 +85,13 @@ func printStatus(asked answers) error {
 	// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
 	// first, silenced ones with when their silence ends.
 	printConditions(asked.conditions, asked.conditionsUnread, time.Now())
+	if len(asked.notAnnounced) > 0 {
+		fmt.Printf("%d module(s) others read the state of have not said what they keep, and are read from `reads` as "+
+			"before until they do (ADR 0495): %s\n\n", len(asked.notAnnounced), strings.Join(asked.notAnnounced, ", "))
+	}
+	if asked.notAnnouncedWhy != "" {
+		fmt.Printf("whose states are read without an announcement cannot be said: %s\n\n", asked.notAnnouncedWhy)
+	}
 
 	if len(asked.refused) > 0 {
 		// First of what follows. A machine that cannot be worked out is not running an old
@@ -435,6 +442,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 	if err != nil {
 		return answers{}, err
 	}
+	// And whose states are still read without an announcement (novox/hq ADR 0495, issue 1790): said, never a reason
+	// for the rest of the answer to fail.
+	if out.notAnnounced, err = notYetAnnounced(ctx, inv); err != nil {
+		out.notAnnounced, out.notAnnouncedWhy = nil, err.Error()
+	}
 	// And what each machine is holding rather than running, by the module that would run it. Read
 	// from what the machine itself last reported, not from what take-time computed: the machine is
 	// the only thing that knows what it found (novox/hq 04-ISSUES/125).