The bundle and the mesh stop removing each other

04-ISSUES/010. The store now records where each resource came from -- carried,
or declared -- and each origin removes only its own. A declaration removes what
the mesh previously declared and never what the bundle raised.

State written before the field existed reads as carried, because everything a
host had applied by then came from its bundle: there was no other way to tell
it anything. Guessing the other way would have the first upgrade remove the
substrate, which is this fault arriving through the change that fixes it.

Verified on the scenario that caused it, and on the property that had to
survive it: a later declaration dropping a resource still removes that
resource, so removal by omission still means what it meant.

Also stops swallowing a publish failure. A node that applied a declaration and
could not tell the mesh looked exactly like one that had -- the mesh believing
it never answered, the node believing it did, and nothing anywhere saying so.
Reports are published mandatory now, so anything the broker cannot route comes
back and is said out loud rather than dropped in silence.

internal/link/run.go

@@ -31,13 +31,20 @@ type Membership struct {
 // Applier is what the host does with a declaration that has been proved to come from the mesh.
 type Applier func(ctx context.Context, declaration []byte) Report
 
+// Announce is how the link says what is happening, so a node running unattended leaves an
+// account of it. Nil is allowed and means say nothing.
+type Announce func(string)
+
 // Run holds the link open, applying what arrives and reporting what happened.
 //
 // Outbound only, and nothing listens on this machine. The connection is the node's presence in
 // the mesh: while it is up the node is enrolled, and while it is down the node is disconnected —
 // which is an ordinary situation and not a failure, so this returns rather than panicking and
 // leaves restarting to whatever supervises it.
-func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration) error {
+func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error {
+	if say == nil {
+		say = func(string) {}
+	}
 	config, err := PinnedConfig(m.Fingerprint)
 	if err != nil {
 		return err
@@ -84,6 +91,18 @@ func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration
 	}
 	closed := conn.NotifyClose(make(chan *amqp.Error, 1))
 
+	// Published mandatory, so the broker hands back anything it cannot route rather than
+	// dropping it. Without this a report goes to an exchange with no matching binding, the
+	// publisher is told nothing, and the mesh believes this node never answered while the node
+	// believes it did — which is what happened when `report` was left unbound on the other side.
+	returned := channel.NotifyReturn(make(chan amqp.Return, 4))
+	go func() {
+		for r := range returned {
+			say(fmt.Sprintf("the broker could not route this node's %s: %s (%d %s)",
+				r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText))
+		}
+	}()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -95,7 +114,15 @@ func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration
 				return errors.New("the broker stopped delivering")
 			}
 			report := handle(ctx, m, apply, delivery)
-			publishReport(ctx, channel, m, report, timeout)
+			switch {
+			case report.Refused != "":
+				say("refused a declaration: " + report.Refused)
+			case len(report.Failed) > 0:
+				say(fmt.Sprintf("applied %d and failed: %v", len(report.Applied), report.Failed))
+			default:
+				say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
+			}
+			publishReport(ctx, channel, m, report, say, timeout)
 			// Acknowledged after the report is published. A node that dies between applying and
 			// reporting leaves the declaration on the broker and applies it again on return,
 			// which is safe because applying is reconciliation — it converges rather than
@@ -127,14 +154,21 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R
 }
 
 func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
-	timeout time.Duration) {
+	say Announce, timeout time.Duration) {
 	report.Node = m.Node
 	body, err := json.Marshal(report)
 	if err != nil {
+		say("cannot encode this node's own report: " + err.Error())
 		return
 	}
 	publish, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	_ = channel.PublishWithContext(publish, Exchange, KeyReport, false, false,
-		amqp.Publishing{ContentType: "application/json", Body: body})
+
+	// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
+	// node never answered, while the node believes it did — and the two would go on disagreeing
+	// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
+	if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false,
+		amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
+		say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
+	}
 }