A service the mesh asked to run is still running a moment later (hq ADR 0184)

The read-back raced the failure: a service manager returns when it has started the process, and a
daemon that refuses its configuration exits a fraction of a second later, so one look saw it alive.
fail2ban took 221ms on the control node and the apply reported "restarted" onto a dead daemon while
both public machines kept no bans at all. The host looks again, after that moment. A unit still
starting is accepted at both looks; a service asked to stop is not waited on.

internal/apply/stayed_running_test.go

@@ -0,0 +1,91 @@
+package apply
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A daemon that reads a configuration it refuses dies a fraction of a second after the service
+// manager has reported it started — fail2ban took 221 milliseconds on the control node the day this
+// was written. One read back catches nothing: the unit is active at that instant. The mesh reported
+// the service "restarted" while every ban on two public machines was gone, and every check passed
+// (novox/hq ADR 0184). The host looks again, after the moment in which that happens.
+func TestAServiceThatDiesJustAfterItsRestartIsNotReportedRestarted(t *testing.T) {
+	serviceSettle = 0
+	// Alive at the first look after starting, dead at the second — the shape of a daemon that
+	// refuses what it was just given.
+	started, looks := false, 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			state := "active"
+			if started {
+				if looks++; looks >= 2 {
+					state = "failed"
+				}
+			}
+			return "LoadState=loaded\nActiveState=" + state + "\n", nil
+		}
+		if args[0] == "start" {
+			started = true
+		}
+		return "", nil
+	}
+	dir := t.TempDir()
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"conf","type":"file","path":"`+dir+`/jail.conf","content":"[sshd]\n","mode":"0644"},
+	  {"id":"run","type":"service","unit":"fail2ban.service","state":"running","restart-on":["conf"]}
+	]}`)
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err == nil {
+		t.Fatal("a service that died just after being restarted was reported as restarted")
+	}
+	if !strings.Contains(err.Error(), "fail2ban.service") || !strings.Contains(err.Error(), "is stopped") {
+		t.Errorf("the failure does not name the unit and what it is now: %v", err)
+	}
+	if looks < 2 {
+		t.Errorf("the host looked at the unit %d time(s) after starting it; it must look again", looks)
+	}
+}
+
+// A unit still coming up reads as running at both looks and is accepted: the second look is for a
+// unit that WAS running and is not any more, never a wait for a slow one to finish starting.
+func TestAUnitStillStartingIsNotAFailure(t *testing.T) {
+	serviceSettle = 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			return "LoadState=loaded\nActiveState=activating\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"slow.service","state":"running"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("a unit still starting was reported as a failure: %v", err)
+	}
+}
+
+// And a service the declaration asks to be stopped is not waited on at all.
+func TestAServiceAskedToStopIsNotWaitedOn(t *testing.T) {
+	serviceSettle = 0
+	shows := 0
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "show" {
+			shows++
+			if shows == 1 {
+				return "LoadState=loaded\nActiveState=active\n", nil
+			}
+			return "LoadState=loaded\nActiveState=inactive\n", nil
+		}
+		return "", nil
+	}
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"s","type":"service","unit":"off.service","state":"stopped"}
+	]}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
+		t.Fatalf("stopping a service was reported as a failure: %v", err)
+	}
+}