173 lines
7.1 KiB
Go
173 lines
7.1 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go"
|
|
)
|
|
|
|
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
|
|
//
|
|
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
|
|
// the same kind of statement and the difference is the whole of this interface — one must arrive
|
|
// and one must not be insisted on.
|
|
//
|
|
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
|
|
// interface over its own client libraries, not a contract shared with the controller. The two
|
|
// agree because a conformance fixture holds them to one envelope, which is the only kind of
|
|
// agreement that survives being in different repositories.
|
|
type Bus interface {
|
|
// Report says what this node applied. **It must arrive.** A report that fails leaves the
|
|
// mesh believing the node never answered while the node believes it did, and the two go on
|
|
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
|
|
// finding. Returns false when it could not be delivered, so the caller can say so.
|
|
Report(ctx context.Context, node string, body []byte) error
|
|
|
|
// Alive says this node is here, and nothing else. **Losing one is nothing**: the next is a
|
|
// minute away and the mesh reads a gap rather than counting arrivals. Insisting on delivery
|
|
// would turn a harmless miss into a logged failure every minute.
|
|
Alive(ctx context.Context, node string, body []byte) error
|
|
}
|
|
|
|
// --- The bus the mesh runs on today -----------------------------------------------------------
|
|
|
|
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
|
|
// the controller's store restarting; a heartbeat does not, because it must not.
|
|
type OverNATS struct {
|
|
Conn *nats.Conn
|
|
JS nats.JetStreamContext
|
|
}
|
|
|
|
// ReportSubject and AliveSubject are this node's own, and no other node's: a host's account may
|
|
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
|
|
// which node a report is about.
|
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
|
|
|
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
|
|
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
|
|
// statement lost is said again within a minute while anything is not healthy.
|
|
func HealthSubject(node string) string { return "mesh.control." + node + ".health" }
|
|
|
|
// HealthBus is a link that can say a health statement. Separate from Bus so a link that cannot is still
|
|
// a link: the statement then waits for the next report, which carries it too.
|
|
type HealthBus interface {
|
|
Health(ctx context.Context, node string, body []byte) error
|
|
}
|
|
|
|
// Health says a statement, core and flushed, as a heartbeat is.
|
|
func (b OverNATS) Health(ctx context.Context, node string, body []byte) error {
|
|
if err := b.Conn.Publish(HealthSubject(node), body); err != nil {
|
|
return err
|
|
}
|
|
flush, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
return b.Conn.FlushWithContext(flush)
|
|
}
|
|
|
|
// CLISubject is where this node's engine asks the controller a line `mesh-cli` was given on this machine
|
|
// (novox/hq ADR 0272 §3): its own node's, inside the grant every engine already has (`mesh.control.<node>.>`), so the
|
|
// bus server is the authority on which node asked. Core NATS, never the CONTROL stream, which names its own subjects.
|
|
func CLISubject(node string) string { return "mesh.control." + node + ".cli" }
|
|
|
|
// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
|
|
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
|
|
func ToolSubject(module, tool, node string) string {
|
|
return "mesh.mod." + module + ".tool." + tool + "." + node
|
|
}
|
|
|
|
// ToolBus is a link that can ask a module's tool and wait for its answer.
|
|
type ToolBus interface {
|
|
Ask(ctx context.Context, subject string, body []byte) ([]byte, error)
|
|
}
|
|
|
|
// Ask asks on core NATS and waits for the answer, within ctx.
|
|
func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte, error) {
|
|
msg, err := b.Conn.RequestWithContext(ctx, subject, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return msg.Data, nil
|
|
}
|
|
|
|
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
|
|
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
|
|
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
|
|
// the message is redelivered; a publish the bus never accepted must fail here rather than
|
|
// be assumed.
|
|
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
|
|
return fmt.Errorf("reporting: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b OverNATS) Alive(ctx context.Context, node string, body []byte) error {
|
|
// Core, deliberately: a heartbeat in a stream is the mesh's least valuable message competing
|
|
// for retention with its most valuable, and a lost one is the next one.
|
|
if err := b.Conn.Publish(AliveSubject(node), body); err != nil {
|
|
return err
|
|
}
|
|
// Flushed rather than fired and forgotten, so "could not tell the mesh" means the write
|
|
// failed rather than that nobody has looked yet.
|
|
flush, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
return b.Conn.FlushWithContext(flush)
|
|
}
|
|
|
|
// HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes.
|
|
// **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket
|
|
// is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every
|
|
// call on the bucket waits no longer than ctx has left: a refused publish is answered by nothing, and the client's own
|
|
// wait is five seconds.
|
|
func (b OverNATS) HaveBind(ctx context.Context) (HaveBucket, error) {
|
|
js := b.JS
|
|
wait := 5 * time.Second
|
|
if at, ok := ctx.Deadline(); ok {
|
|
wait = time.Until(at)
|
|
if wait <= 0 {
|
|
return nil, context.DeadlineExceeded
|
|
}
|
|
}
|
|
if b.Conn != nil {
|
|
bounded, err := b.Conn.JetStream(nats.MaxWait(wait))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
js = bounded
|
|
}
|
|
kv, err := js.KeyValue(delivery.HaveBucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the have bucket %s: %w", delivery.HaveBucket, err)
|
|
}
|
|
return natsHaveBucket{kv}, nil
|
|
}
|
|
|
|
type natsHaveBucket struct{ kv nats.KeyValue }
|
|
|
|
func (n natsHaveBucket) Get(_ context.Context, key string) ([]byte, uint64, error) {
|
|
entry, err := n.kv.Get(key)
|
|
if errors.Is(err, nats.ErrKeyNotFound) {
|
|
return nil, 0, ErrHaveAbsent
|
|
}
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
return entry.Value(), entry.Revision(), nil
|
|
}
|
|
|
|
func (n natsHaveBucket) Set(_ context.Context, key string, value []byte, revision uint64) error {
|
|
var err error
|
|
if revision == 0 {
|
|
_, err = n.kv.Create(key, value)
|
|
return err
|
|
}
|
|
_, err = n.kv.Update(key, value, revision)
|
|
return err
|
|
}
|
|
|
|
var _ HaveBus = OverNATS{}
|