Keep serving the consumers postgres's grant file gives on its moved path (issue 937)

Only a moved consumer's plan names postgres, so on the moved path every
consumer still given by the controller's grant file was counted unasked
and offered for retirement (ace 5 of 5, novox 7 of 9), and could have
been retired unasked where the bound allowed. The moved path now reads
MESH_RECEIVES again: its logins are asked, the ones no plan names are
made and checked by the harness's grant pass until R1 (issue 682), and
an unreadable grant file retires nothing (issue 241). The harness's
grant pass becomes apply, identically in keycloak's copy.

modules/keycloak/cmd/keycloak-provider/harness.go

@@ -321,6 +321,29 @@ func (h *Harness) Reconcile(ctx context.Context) {
 		want[g.As] = true
 	}
 	h.seed(ctx, want)
+	h.apply(ctx, given)
+
+	// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
+	h.retireUnasked(ctx, want)
+	h.r.lastWant = want
+	for as := range h.failing {
+		if !want[as] {
+			delete(h.failing, as)
+		}
+	}
+	// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
+	// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
+	// consumer that is gone. One still held is said recovered when it is retired.
+	for as := range h.trouble {
+		if _, held := h.applied[as]; !want[as] && !held {
+			h.recovered(as, "no longer asked for")
+		}
+	}
+}
+
+// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The
+// caller holds h.mu. One consumer's failure never stops the others'.
+func (h *Harness) apply(ctx context.Context, given []contribution) {
 	verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery
 	if verifying {
 		h.verifiedAt = h.Now()
@@ -423,23 +446,6 @@ func (h *Harness) Reconcile(ctx context.Context) {
 			}
 		}
 	}
-
-	// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
-	h.retireUnasked(ctx, want)
-	h.r.lastWant = want
-	for as := range h.failing {
-		if !want[as] {
-			delete(h.failing, as)
-		}
-	}
-	// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
-	// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
-	// consumer that is gone. One still held is said recovered when it is retired.
-	for as := range h.trouble {
-		if _, held := h.applied[as]; !want[as] && !held {
-			h.recovered(as, "no longer asked for")
-		}
-	}
 }
 
 // failed counts one more failure in a consumer's unbroken run, and announces the run once it has

modules/postgres/cmd/postgres-provider/grants_moved_test.go

@@ -0,0 +1,123 @@
+package main
+
+// postgres handed to the provisioner still serves the consumers the controller's grant file gives (issue 937): a
+// consumer not moved has no plan naming postgres, only its grant, until R1 (issue 682).
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+)
+
+// granting gives the moved world a grant file, as the controller writes it for the consumers not moved.
+func (w *movedWorld) granting(given ...map[string]any) {
+	if w.h.Receives == "" {
+		w.h.Receives = filepath.Join(w.t.TempDir(), "mesh.json")
+	}
+	dir := filepath.Dir(w.h.Receives)
+	for _, g := range given {
+		secret := filepath.Join(dir, g["as"].(string)+".secret")
+		if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil {
+			w.t.Fatal(err)
+		}
+		g["secret"] = secret
+	}
+	if given == nil {
+		given = []map[string]any{}
+	}
+	raw, _ := json.Marshal(map[string]any{"requirement": ProvisionName, "given": given})
+	if err := os.WriteFile(w.h.Receives, raw, 0o600); err != nil {
+		w.t.Fatal(err)
+	}
+}
+
+// settled is enough passes and time for a retirement to be stable.
+func (w *movedWorld) settled() {
+	for i := 0; i < 8; i++ {
+		w.pass()
+	}
+	w.now = w.now.Add(15 * time.Minute)
+	w.pass()
+	w.pass()
+}
+
+// The logins the grant file gives are asked for: made as the grant path makes them, never offered for retirement,
+// beside a moved consumer the helper makes from its plan.
+func TestAConsumerOnlyTheGrantFileGivesIsAskedForAndMade(t *testing.T) {
+	bound, logins := bindings(1)
+	w := newMovedWorld(t, bound)
+	w.a.inv.Active = []string{"mesh_ace_baserow", "mesh_ace_letta", logins[0]}
+	w.bus.bind("ace", "app1", nil)
+	w.granting(
+		map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}},
+		map[string]any{"as": "mesh_ace_letta", "node": "ace",
+			"values": map[string]any{"name": "letta", "extensions": []any{"vector"}}},
+	)
+	w.settled()
+	if len(w.a.removed) != 0 || w.h.r.waiting != nil {
+		t.Fatalf("retired %v, waiting %+v: the grant file's logins were counted unasked", w.a.removed, w.h.r.waiting)
+	}
+	made := map[string]Provision{}
+	for _, c := range w.a.created {
+		made[c.As] = c
+	}
+	letta := made["mesh_ace_letta"]
+	if letta.Password != "pw-mesh_ace_letta" || letta.Consumer != "ace" ||
+		!reflect.DeepEqual(letta.Values, map[string]any{"name": "letta", "extensions": []any{"vector"}}) {
+		t.Fatalf("made %+v for letta, as its grant gives it", letta)
+	}
+	if _, ok := made["mesh_ace_baserow"]; !ok {
+		t.Fatalf("baserow not made: %+v", w.a.created)
+	}
+	if _, ok := made[logins[0]]; !ok {
+		t.Fatalf("the moved consumer not made: %+v", w.a.created)
+	}
+}
+
+// A login both a plan and the grant file name is the plan's: made once, with the plan's credential.
+func TestALoginAPlanNamesIsNotMadeFromTheGrantFile(t *testing.T) {
+	bound, logins := bindings(1)
+	w := newMovedWorld(t, bound)
+	w.bus.bind("ace", "app1", nil)
+	w.granting(map[string]any{"as": logins[0], "node": "ace", "values": map[string]any{"name": "app1"}})
+	w.pass()
+	w.pass()
+	for _, c := range w.a.created {
+		if c.As == logins[0] && c.Password != "pw-app1" {
+			t.Fatalf("made %s with the grant's password %q, not the plan's", c.As, c.Password)
+		}
+	}
+}
+
+// An unreadable grant file is not nobody asking (issue 241): nothing is retired while it cannot be read.
+func TestAnUnreadableGrantFileRetiresNothing(t *testing.T) {
+	bound, logins := bindings(1)
+	w := newMovedWorld(t, bound)
+	w.a.inv.Active = []string{"mesh_ace_baserow", logins[0]}
+	w.bus.bind("ace", "app1", nil)
+	w.h.Receives = filepath.Join(t.TempDir(), "absent.json")
+	w.settled()
+	if len(w.a.removed) != 0 || w.h.r.waiting != nil {
+		t.Fatalf("retired %v, waiting %+v with the grant file unreadable", w.a.removed, w.h.r.waiting)
+	}
+}
+
+// A login the grant file stops giving is retired by the usual rules: stable, then retired, never dropped.
+func TestALoginTheGrantFileStopsGivingIsRetired(t *testing.T) {
+	bound, _ := bindings(1)
+	w := newMovedWorld(t, bound)
+	w.bus.bind("ace", "app1", nil)
+	w.granting(
+		map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}},
+		map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}},
+	)
+	w.pass()
+	w.granting(map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}})
+	w.settled()
+	if !reflect.DeepEqual(w.a.removed, []string{"mesh_ace_letta"}) || len(w.a.deleted) != 0 {
+		t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted)
+	}
+}

modules/postgres/cmd/postgres-provider/harness.go

@@ -321,6 +321,29 @@ func (h *Harness) Reconcile(ctx context.Context) {
 		want[g.As] = true
 	}
 	h.seed(ctx, want)
+	h.apply(ctx, given)
+
+	// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
+	h.retireUnasked(ctx, want)
+	h.r.lastWant = want
+	for as := range h.failing {
+		if !want[as] {
+			delete(h.failing, as)
+		}
+	}
+	// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
+	// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
+	// consumer that is gone. One still held is said recovered when it is retired.
+	for as := range h.trouble {
+		if _, held := h.applied[as]; !want[as] && !held {
+			h.recovered(as, "no longer asked for")
+		}
+	}
+}
+
+// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The
+// caller holds h.mu. One consumer's failure never stops the others'.
+func (h *Harness) apply(ctx context.Context, given []contribution) {
 	verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery
 	if verifying {
 		h.verifiedAt = h.Now()
@@ -423,23 +446,6 @@ func (h *Harness) Reconcile(ctx context.Context) {
 			}
 		}
 	}
-
-	// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
-	h.retireUnasked(ctx, want)
-	h.r.lastWant = want
-	for as := range h.failing {
-		if !want[as] {
-			delete(h.failing, as)
-		}
-	}
-	// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
-	// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
-	// consumer that is gone. One still held is said recovered when it is retired.
-	for as := range h.trouble {
-		if _, held := h.applied[as]; !want[as] && !held {
-			h.recovered(as, "no longer asked for")
-		}
-	}
 }
 
 // failed counts one more failure in a consumer's unbroken run, and announces the run once it has

modules/postgres/cmd/postgres-provider/main.go

@@ -35,10 +35,11 @@ func main() {
 	var h *Harness
 	if consumers := handedConsumersFile(os.Getenv, os.Stat); consumers != "" {
 		// Handed to the provisioner (novox/hq ADR 0373): the provider helper answers the consumers its
-		// machine's client fills the file with; the grant directory is not read. Retirement is the
-		// harness's, never run, so its guards and its tools stand (issue 780).
+		// machine's client fills the file with, and the grant file still gives the consumers not moved
+		// (issue 937). Retirement is the harness's, never run, so its guards and its tools stand (issue 780).
 		h = &Harness{
 			Resource: ProvisionName,
+			Receives: os.Getenv("MESH_RECEIVES"),
 			Adapter:  provisioner{pg: pg, announce: announce},
 			Log:      func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
 			Announce: func(event string, body map[string]any) {

modules/postgres/cmd/postgres-provider/moved.go

@@ -8,6 +8,12 @@ package main
 // here: Make brings the consumer's database and role into being exactly as the harness's adapter does, and Gone
 // retires what was kept for a consumer that no longer binds, the reversible way (retire_pg.go): the login is
 // locked and marked to delete, nothing is dropped.
+//
+// **A consumer not moved is still the grant file's** (issue 937, ADR 0349 clause 17): only a moved consumer's plan
+// names its provider, so the controller still gives every other consumer through postgres's grant file
+// (MESH_RECEIVES) until R1 (issue 682). Each login it gives is asked for, and one no plan names is made and checked
+// by the harness's grant pass exactly as on the grant path; a login a plan names is the helper's alone. A grant file
+// that cannot be read is not nobody asking (issue 241): nothing is retired until it is read again.
 
 import (
 	"context"
@@ -102,11 +108,14 @@ func (m *movedRun) Gone(_ context.Context, e provider.Entry) error {
 	if as == "" {
 		return nil // refused: nothing was made for it
 	}
-	// The plans as they are now, not as the last retirement pass read them: with none read, nothing is let go.
+	// The plans and the grant file as they are now, not as the last retirement pass read them: with either not
+	// read, nothing is let go.
 	want, read := m.asked(h.Node)
-	asked := want[as]
 	h.mu.Lock()
 	defer h.mu.Unlock()
+	_, granted, ok := m.granted(want)
+	read = read && ok
+	asked := want[as] || granted[as]
 	_, held := h.applied[as]
 	_, retired := h.r.retired[as]
 	switch {
@@ -165,18 +174,50 @@ func (m *movedRun) asked(node string) (map[string]bool, bool) {
 	return want, true
 }
 
-// retire is one pass of the harness's retirement over what the plans bind now.
+// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it
+// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read.
+func (m *movedRun) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) {
+	h := m.h
+	if h.Receives == "" {
+		return nil, map[string]bool{}, true
+	}
+	given := h.readContributions()
+	if given == nil {
+		return nil, nil, false
+	}
+	var own []contribution
+	logins := map[string]bool{}
+	for _, g := range given {
+		logins[g.As] = true
+		if !planned[g.As] {
+			own = append(own, g)
+		}
+	}
+	return own, logins, true
+}
+
+// retire is one pass of the harness's retirement over what the plans bind and the grant file gives now; the logins
+// only the grant file gives are made and checked first, as the grant path does.
 func (m *movedRun) retire(ctx context.Context) {
 	h := m.h
 	want, ok := m.asked(h.Node)
 	h.mu.Lock()
 	defer h.mu.Unlock()
+	var given []contribution
+	if ok {
+		var granted map[string]bool
+		given, granted, ok = m.granted(want)
+		for as := range granted {
+			want[as] = true
+		}
+	}
 	if !ok {
 		// Not a result: the passes that must agree before anything is retired start again.
 		h.r.key, h.r.count = "", 0
 		return
 	}
 	h.seed(ctx, want)
+	h.apply(ctx, given)
 	h.retireUnasked(ctx, want)
 	h.r.lastWant = want
 	for as := range h.failing {