mesh-controller/internal/link/standing.go

package link

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/broker"
)

// A provider's standing (novox/hq ADR 0224).
//
// **A provider that keeps failing a consumer is a problem the controller reports**, not a line in a
// journal. On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a
// day — its admin no longer took the mesh's secret once its database was moved — and every surface
// the mesh has called the mesh well (novox/hq issue 179). A provider now says, as an event, a
// consumer it has failed for minutes without one success, and the consumer recovering; the
// controller keeps the newest word per provider, machine and consumer, and `status` names each one
// still failing.

// Standing is one provider's word about one consumer.
type Standing struct {
	// Module is the emitter, read from the subject the bus let it publish on — never from the body.
	Module string `json:"-"`
	// Failing is which of the two it said: failing, or recovered.
	Failing bool `json:"-"`

	Provider     string    `json:"provider"`
	ProviderNode string    `json:"provider-node"`
	Consumer     string    `json:"consumer"`
	Node         string    `json:"node"`
	Class        string    `json:"class,omitempty"`
	Error        string    `json:"error,omitempty"`
	Since        time.Time `json:"since"`
	Attempts     int       `json:"attempts"`
	// Why is said with a recovery that is not a success: `withdrawn`, a consumer no longer asked for.
	Why string `json:"why,omitempty"`
}

// Standings keeps what providers say about their consumers.
type Standings interface {
	// Stood records a provider's newest word about a consumer: a failing one kept, a recovered one
	// cleared — and says whether a recovery cleared anything, since a provider announces its first
	// success for every consumer after it starts. An error the store is away for is held and asked
	// again, like a report.
	Stood(ctx context.Context, s Standing) (cleared bool, err error)
}

// Watches says where providers' standings are kept, and asks for them to be delivered.
func (s *Server) Watches(st Standings) error {
	if err := s.inbound.Also(KindProvisioner); err != nil {
		return err
	}
	s.standings = st
	return nil
}

// ReadStanding is one standing event as the controller understands it, from its subject and body.
func ReadStanding(subject string, body []byte) (Standing, error) {
	module, ok := ProvisionerEmitter(subject)
	if !ok {
		return Standing{}, fmt.Errorf("%s is not a provider's standing", subject)
	}
	var st Standing
	if err := json.Unmarshal(body, &st); err != nil {
		return Standing{}, fmt.Errorf("%s's standing could not be read: %w", module, err)
	}
	if st.Consumer == "" {
		return Standing{}, fmt.Errorf("%s's standing named no consumer", module)
	}
	st.Module = module
	st.Failing = strings.HasSuffix(subject, "."+broker.ProvisionerFailing)
	return st, nil
}

// provisioner acts on one standing event.
//
// **A recovery must not be lost.** A failing standing is said again every quarter of an hour while
// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
func (s *Server) provisioner(ctx context.Context, m Control) {
	if IsRetirement(m.Subject()) {
		s.retirement(ctx, m)
		return
	}
	if s.standings == nil {
		// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
		// because handing it back would not give it anywhere to go.
		_ = m.Took()
		return
	}
	st, err := ReadStanding(m.Subject(), m.Body())
	if err != nil {
		s.log.Printf("%v; ignored", err)
		_ = m.Took()
		return
	}
	cleared, err := s.standings.Stood(ctx, st)
	what := fmt.Sprintf("%s's standing for %s", st.Module, st.Consumer)
	switch s.decide(ctx, m, what, "", "", err) {
	case Hold:
		return
	case Stale, GiveUp:
		_ = m.Took()
		return
	}
	if err != nil {
		s.log.Printf("%s could not be kept: %v", what, err)
	} else if st.Failing {
		s.log.Printf("%s on %s is FAILING %s on %s (%s, %d attempts since %s): %s", st.Module,
			st.ProviderNode, st.Consumer, st.Node, st.Class, st.Attempts, st.Since.Format(time.RFC3339), st.Error)
	} else if cleared {
		s.log.Printf("%s on %s recovered %s", st.Module, st.ProviderNode, st.Consumer)
	}
	_ = m.Took()
}