Merge pull request 'Refuse a declaration composed from an older assignment generation than the machine applied (issue 234)' (#82) from fix/234-refuse-a-declaration-from-an-older-generation into main

internal/declaration/declaration.go

@@ -1378,6 +1378,20 @@ type Declaration struct {
 	// what it wrote for it — its resources are absent from the declaration, and absence would
 	// otherwise read as removal.
 	LeftOut []string
+
+	// Generation is the assignment generation this declaration was composed from: the controller's
+	// counter, raised in the same transaction as every change to what is assigned where (novox/hq
+	// issue 234). The sequence orders arrival and cannot tell a later send that carries an older view
+	// of the assignments; this can. A node-engine refuses a declaration composed from a generation
+	// older than the highest it applied — unless it is a put-back — because applying it would
+	// undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none,
+	// and is applied as before.
+	Generation int64
+
+	// PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries
+	// the generation of what it puts back, which may be older than what the machine applied, and is
+	// not refused for it.
+	PutBack bool
 }
 
 // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
@@ -1542,6 +1556,11 @@ type envelope struct {
 	Epoch int64 `json:"epoch,omitempty"`
 	// LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163).
 	LeftOut []string `json:"left_out,omitempty"`
+	// Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller
+	// that claims no generation. **An older host refuses these keys**, decoding strictly; a controller
+	// sends them only to a host whose reports say `reads_generation`.
+	Generation int64 `json:"generation,omitempty"`
+	PutBack    bool  `json:"put_back,omitempty"`
 }
 
 func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1559,8 +1578,17 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
 	}
 
 	d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
-		Epoch: env.Epoch, LeftOut: env.LeftOut}
+		Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
 	var problems []string
+	if env.Generation < 0 {
+		// Below zero would read as "none claimed" and pass every refusal of an older generation.
+		problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
+			"mesh assigns", env.Generation))
+	}
+	if env.PutBack && allowActions {
+		// A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back.
+		problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that")
+	}
 	if env.Sequence < 0 || env.Epoch < 0 {
 		// Below zero is no order any controller assigns, and read as "none claimed" it would let the
 		// declaration past every refusal of what is older.

internal/declaration/order_test.go

@@ -35,3 +35,25 @@ func TestAnOrderBelowZeroIsRefused(t *testing.T) {
 		}
 	}
 }
+
+// **The assignment generation it was composed from, and whether it is a put-back** (novox/hq issue 234):
+// absent is no claim; below zero is none the mesh assigns; a carried bundle puts nothing back.
+func TestADeclarationCarriesItsGenerationAndWhetherItIsAPutBack(t *testing.T) {
+	d, err := Parse([]byte(withOrder(`"sequence":3,"generation":41,"put_back":true,`)))
+	if err != nil {
+		t.Fatalf("a declaration carrying its generation was refused: %v", err)
+	}
+	if d.Generation != 41 || !d.PutBack {
+		t.Fatalf("read generation %d, put-back %v", d.Generation, d.PutBack)
+	}
+	if d, err = Parse([]byte(withOrder(``))); err != nil || d.Generation != 0 || d.PutBack {
+		t.Fatalf("a declaration with no generation read as %+v, %v", d, err)
+	}
+	if refusal := refusalFor(t, withOrder(`"generation":-2,`)); !strings.Contains(refusal.Error(), "below zero") {
+		t.Errorf("a generation below zero was refused for something else: %v", refusal)
+	}
+	if _, err := ParseTrusted([]byte(withOrder(`"generation":3,"put_back":true,`))); err == nil ||
+		!strings.Contains(err.Error(), "put-back") {
+		t.Errorf("a carried bundle said it was a put-back and was not refused for it: %v", err)
+	}
+}

internal/link/messages.go

@@ -128,6 +128,17 @@ type Report struct {
 	// read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts.
 	RefusedOlder int64 `json:"refused_older,omitempty"`
 
+	// ReadsGeneration says this node-engine reads a declaration's assignment generation and its
+	// put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a
+	// machine that said so, because an older node-engine decodes strictly and refuses a key it does not
+	// know, whole. A node-engine rolled back says so no longer.
+	ReadsGeneration bool `json:"reads_generation,omitempty"`
+
+	// OlderGeneration is set on a report refusing a declaration composed from an older assignment
+	// generation than this machine applied (novox/hq issue 234). The refused declaration is the report's
+	// own Declared and Order, so the controller can name who sent it from its record of sends.
+	OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"`
+
 	// Held is what this adopted node found and is keeping as it was until its module is taken
 	// (novox/hq ADR 0100). Without it an adopted node reads as converged.
 	Held []Held `json:"held,omitempty"`
@@ -634,3 +645,10 @@ type FoundFirewall struct {
 	Active    bool   `json:"active"`
 	RetiredBy string `json:"retired_by,omitempty"`
 }
+
+// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that
+// generation, and the highest this machine applied (novox/hq issue 234).
+type GenerationRefusal struct {
+	Generation int64 `json:"generation"`
+	Applied    int64 `json:"applied"`
+}

internal/link/queue.go

@@ -211,6 +211,12 @@ func (q *Queue) step(ctx context.Context) bool {
 		q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+
 			"holder — %d refused as older so far", short(report.Declared), report.Order.Words(),
 			report.OlderThan.Words(), total))
+	case report.OlderGeneration != nil:
+		// Said here, and the report names it, so the controller raises it naming the sender (novox/hq
+		// issue 234). Not counted with the lease's refusals: those are S13's, by epoch.
+		q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+
+			"this node applied generation %d", short(report.Declared), report.Order.Words(),
+			report.OlderGeneration.Generation, report.OlderGeneration.Applied))
 	case report.Refused != "":
 		q.Say("refused a declaration: " + report.Refused)
 	case len(report.Failed) > 0:
@@ -228,7 +234,7 @@ func (q *Queue) step(ctx context.Context) bool {
 	// A refusal as older is not kept to be said again: what the mesh waits for from this node is the
 	// account of the last apply, and that is still what is kept.
 	kind := applied
-	if report.OlderThan != nil {
+	if report.OlderThan != nil || report.OlderGeneration != nil {
 		kind = refusedOlder
 	}
 	q.tell(ctx, report, kind, latest)
@@ -336,6 +342,9 @@ func (q *Queue) stamp(r Report) Report {
 	r.Node = q.Membership.Node
 	r.ReportSequence = q.numbers.sequence
 	r.RefusedOlder = q.numbers.refused
+	// Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue
+	// 234): this build does, and the controller sends one only to a machine that said so.
+	r.ReadsGeneration = true
 	return r
 }
 

internal/link/queue_test.go

@@ -540,3 +540,26 @@ func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) {
 		t.Fatalf("a report was said on a link let go: %+v", l.said())
 	}
 }
+
+// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the
+// controller sends one only to a machine that said so, because an older node-engine refuses the key.
+func TestEveryReportSaysItReadsAGeneration(t *testing.T) {
+	m, key := aMember(t)
+	l := newQuietLink()
+	a := &applies{}
+	q := aQueue(m, a.apply, &keptInMemory{})
+	q.attach(context.Background(), l)
+	q.Deliver(ordered(t, key, "new", 7, 12))
+	ctx, stop := context.WithCancel(context.Background())
+	worker := runWorker(ctx, q)
+	eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
+	stop()
+	<-worker
+	raw, err := json.Marshal(l.said()[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(raw), `"reads_generation":true`) {
+		t.Fatalf("the report does not say it reads a generation: %s", raw)
+	}
+}

internal/store/store.go

@@ -187,6 +187,12 @@ type Into struct {
 
 // State is the whole of what a node knows about what it has done.
 type State struct {
+	// Generation is the highest assignment generation this machine applied a declaration from
+	// (novox/hq issue 234): a declaration from an older one is refused unless it is a put-back. Only
+	// ever raised — a put-back of an older generation leaves it — so a put-back cannot open the door to
+	// the stale send it protects against. Zero on a machine never sent one.
+	Generation int64 `json:"generation,omitempty"`
+
 	// Resources, keyed by identity, in the order they were applied. Order matters for removal:
 	// undoing in reverse is the only ordering the host can derive without deciding anything.
 	Resources []Applied `json:"resources"`