mesh-controller/cmd/mesh-controller/apply_failing_test.go

package main

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
)

// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
// the same failure on times reports since for before now.
func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
	since := now.Add(-forHowLong)
	return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
		Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
			{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
				"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
				"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
			{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
				"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
				"file /etc/pacman.conf could not be read: No such file or directory"}}}
}

// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
	f := calm(now)
	f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
	got := watchApplyFailing(f)
	if len(got) != 1 {
		t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
	}
	o := got[0]
	if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
		t.Errorf("the condition is not the machine's: %s", o.Key())
	}
	if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
		t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
	}
	if strings.Contains(o.Summary, "/etc/") {
		t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
	}
	if !strings.Contains(o.Said, "could not be read") {
		t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
	}
	if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
		t.Errorf("the plain words do not say it: %+v", o)
	}
}

// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
// node-engine applied the rest.
func TestARefusalThatGoesOnIsRaised(t *testing.T) {
	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
	since := now.Add(-time.Hour)
	f := calm(now)
	f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
		Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
	got := watchApplyFailing(f)
	if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
		!strings.Contains(got[0].Said, "pacman does not answer") {
		t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
	}
}

// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
// younger is not yet; and a silent or sleeping machine is S1's.
func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
	f := calm(now)
	f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
	if got := watchApplyFailing(f); len(got) != 1 {
		t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
	}
	f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
	if got := watchApplyFailing(f); len(got) != 0 {
		t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
	}
	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
	f.machines[0].lastHeard = now.Add(-time.Hour)
	if got := watchApplyFailing(f); len(got) != 0 {
		t.Errorf("a silent machine was raised for its last report: %+v", got)
	}
	f = calm(now)
	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
	f.doing[0].Node = "a-machine-the-mesh-does-not-know"
	if got := watchApplyFailing(f); len(got) != 0 {
		t.Errorf("a machine not heard of was raised: %+v", got)
	}
}

// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
// is cleared through the keeper on the next tick.
func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
		Teller: &conditions.Told{}, Now: func() time.Time { return now }})
	defer k.Close(context.Background())
	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}

	failing := calm(now)
	failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
	w.see(t.Context(), failing)
	open, err := k.Open(t.Context())
	if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
		t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
	}

	clean := calm(now.Add(5 * time.Minute))
	clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
	w.see(t.Context(), clean)
	if open, _ := k.Open(t.Context()); len(open) != 0 {
		t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
	}
}