mesh-host/internal/link/bus.go

package link

import (
	"context"
	"errors"
	"fmt"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"github.com/nats-io/nats.go"
)

// Bus is what a host needs of the mesh's bus, in the mesh's own words.
//
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
// the same kind of statement and the difference is the whole of this interface — one must arrive
// and one must not be insisted on.
//
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
// interface over its own client libraries, not a contract shared with the controller. The two
// agree because a conformance fixture holds them to one envelope, which is the only kind of
// agreement that survives being in different repositories.
type Bus interface {
	// Report says what this node applied. **It must arrive.** A report that fails leaves the
	// mesh believing the node never answered while the node believes it did, and the two go on
	// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
	// finding. Returns false when it could not be delivered, so the caller can say so.
	Report(ctx context.Context, node string, body []byte) error

	// Alive says this node is here, and nothing else. **Losing one is nothing**: the next is a
	// minute away and the mesh reads a gap rather than counting arrivals. Insisting on delivery
	// would turn a harmless miss into a logged failure every minute.
	Alive(ctx context.Context, node string, body []byte) error
}

// --- The bus the mesh runs on today -----------------------------------------------------------

// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
// the controller's store restarting; a heartbeat does not, because it must not.
type OverNATS struct {
	Conn *nats.Conn
	JS   nats.JetStreamContext
}

// ReportSubject and AliveSubject are this node's own, and no other node's: a host's account may
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
// which node a report is about.
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
func AliveSubject(node string) string  { return "mesh.control." + node + ".alive" }

// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
// statement lost is said again within a minute while anything is not healthy.
func HealthSubject(node string) string { return "mesh.control." + node + ".health" }

// HealthBus is a link that can say a health statement. Separate from Bus so a link that cannot is still
// a link: the statement then waits for the next report, which carries it too.
type HealthBus interface {
	Health(ctx context.Context, node string, body []byte) error
}

// Health says a statement, core and flushed, as a heartbeat is.
func (b OverNATS) Health(ctx context.Context, node string, body []byte) error {
	if err := b.Conn.Publish(HealthSubject(node), body); err != nil {
		return err
	}
	flush, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()
	return b.Conn.FlushWithContext(flush)
}

// CLISubject is where this node's engine asks the controller a line `mesh-cli` was given on this machine
// (novox/hq ADR 0272 §3): its own node's, inside the grant every engine already has (`mesh.control.<node>.>`), so the
// bus server is the authority on which node asked. Core NATS, never the CONTROL stream, which names its own subjects.
func CLISubject(node string) string { return "mesh.control." + node + ".cli" }

// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
func ToolSubject(module, tool, node string) string {
	return "mesh.mod." + module + ".tool." + tool + "." + node
}

// ToolBus is a link that can ask a module's tool and wait for its answer.
type ToolBus interface {
	Ask(ctx context.Context, subject string, body []byte) ([]byte, error)
}

// Ask asks on core NATS and waits for the answer, within ctx.
func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte, error) {
	msg, err := b.Conn.RequestWithContext(ctx, subject, body)
	if err != nil {
		return nil, err
	}
	return msg.Data, nil
}

func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
	// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
	// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
	// the message is redelivered; a publish the bus never accepted must fail here rather than
	// be assumed.
	if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
		return fmt.Errorf("reporting: %w", err)
	}
	return nil
}

func (b OverNATS) Alive(ctx context.Context, node string, body []byte) error {
	// Core, deliberately: a heartbeat in a stream is the mesh's least valuable message competing
	// for retention with its most valuable, and a lost one is the next one.
	if err := b.Conn.Publish(AliveSubject(node), body); err != nil {
		return err
	}
	// Flushed rather than fired and forgotten, so "could not tell the mesh" means the write
	// failed rather than that nobody has looked yet.
	flush, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()
	return b.Conn.FlushWithContext(flush)
}

// haveBucket binds to the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132). **Only binds**: a node-engine's
// account reaches no part of the JetStream API beyond what it is granted, and the bucket is the controller's to create
// and to configure, so a bucket that is not there is an error said as what it is.
func (b OverNATS) haveBucket() (nats.KeyValue, error) {
	kv, err := b.JS.KeyValue(delivery.HaveBucket)
	if err != nil {
		return nil, fmt.Errorf("the have bucket %s: %w", delivery.HaveBucket, err)
	}
	return kv, nil
}

// HaveGet reads a key of the have bucket with its revision.
func (b OverNATS) HaveGet(_ context.Context, key string) ([]byte, uint64, error) {
	kv, err := b.haveBucket()
	if err != nil {
		return nil, 0, err
	}
	entry, err := kv.Get(key)
	if errors.Is(err, nats.ErrKeyNotFound) {
		return nil, 0, ErrHaveAbsent
	}
	if err != nil {
		return nil, 0, err
	}
	return entry.Value(), entry.Revision(), nil
}

// HaveSet writes a key of the have bucket over the revision read, or creates it when the revision is zero.
func (b OverNATS) HaveSet(_ context.Context, key string, value []byte, revision uint64) error {
	kv, err := b.haveBucket()
	if err != nil {
		return err
	}
	if revision == 0 {
		_, err = kv.Create(key, value)
		return err
	}
	_, err = kv.Update(key, value, revision)
	return err
}

var _ HaveBus = OverNATS{}