jochen
bdd44154cc
Liveness alone could not see a web application whose port was open and whose program ran while every request hung for eleven hours (issue 145). A resource now carries the `health` its module declared: the engine makes http and tcp looks itself from the machine to the endpoint's published port, reads a unit's readiness from the show it already makes, hands an exec command or the image's own check to the runtime as the container's check with the declared timing and reads its state from the inspect it already makes, and asks a module's tool on its own node tools. Starting until the check passed, unhealthy once its looks after the grace fail the declared number of times; never more looks than the measured budget; nothing restarted. The statement says contract 2, which tells the controller this engine may be sent the field.
113 lines
5.0 KiB
Go
113 lines
5.0 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
)
|
|
|
|
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
|
|
//
|
|
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
|
|
// the same kind of statement and the difference is the whole of this interface — one must arrive
|
|
// and one must not be insisted on.
|
|
//
|
|
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
|
|
// interface over its own client libraries, not a contract shared with the controller. The two
|
|
// agree because a conformance fixture holds them to one envelope, which is the only kind of
|
|
// agreement that survives being in different repositories.
|
|
type Bus interface {
|
|
// Report says what this node applied. **It must arrive.** A report that fails leaves the
|
|
// mesh believing the node never answered while the node believes it did, and the two go on
|
|
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
|
|
// finding. Returns false when it could not be delivered, so the caller can say so.
|
|
Report(ctx context.Context, node string, body []byte) error
|
|
|
|
// Alive says this node is here, and nothing else. **Losing one is nothing**: the next is a
|
|
// minute away and the mesh reads a gap rather than counting arrivals. Insisting on delivery
|
|
// would turn a harmless miss into a logged failure every minute.
|
|
Alive(ctx context.Context, node string, body []byte) error
|
|
}
|
|
|
|
// --- The bus the mesh runs on today -----------------------------------------------------------
|
|
|
|
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
|
|
// the controller's store restarting; a heartbeat does not, because it must not.
|
|
type OverNATS struct {
|
|
Conn *nats.Conn
|
|
JS nats.JetStreamContext
|
|
}
|
|
|
|
// ReportSubject and AliveSubject are this node's own, and no other node's: a host's account may
|
|
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
|
|
// which node a report is about.
|
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
|
|
|
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
|
|
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
|
|
// statement lost is said again within a minute while anything is not healthy.
|
|
func HealthSubject(node string) string { return "mesh.control." + node + ".health" }
|
|
|
|
// HealthBus is a link that can say a health statement. Separate from Bus so a link that cannot is still
|
|
// a link: the statement then waits for the next report, which carries it too.
|
|
type HealthBus interface {
|
|
Health(ctx context.Context, node string, body []byte) error
|
|
}
|
|
|
|
// Health says a statement, core and flushed, as a heartbeat is.
|
|
func (b OverNATS) Health(ctx context.Context, node string, body []byte) error {
|
|
if err := b.Conn.Publish(HealthSubject(node), body); err != nil {
|
|
return err
|
|
}
|
|
flush, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
return b.Conn.FlushWithContext(flush)
|
|
}
|
|
|
|
// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
|
|
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
|
|
func ToolSubject(module, tool, node string) string {
|
|
return "mesh.mod." + module + ".tool." + tool + "." + node
|
|
}
|
|
|
|
// ToolBus is a link that can ask a module's tool and wait for its answer.
|
|
type ToolBus interface {
|
|
Ask(ctx context.Context, subject string, body []byte) ([]byte, error)
|
|
}
|
|
|
|
// Ask asks on core NATS and waits for the answer, within ctx.
|
|
func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte, error) {
|
|
msg, err := b.Conn.RequestWithContext(ctx, subject, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return msg.Data, nil
|
|
}
|
|
|
|
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
|
|
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
|
|
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
|
|
// the message is redelivered; a publish the bus never accepted must fail here rather than
|
|
// be assumed.
|
|
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
|
|
return fmt.Errorf("reporting: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b OverNATS) Alive(ctx context.Context, node string, body []byte) error {
|
|
// Core, deliberately: a heartbeat in a stream is the mesh's least valuable message competing
|
|
// for retention with its most valuable, and a lost one is the next one.
|
|
if err := b.Conn.Publish(AliveSubject(node), body); err != nil {
|
|
return err
|
|
}
|
|
// Flushed rather than fired and forgotten, so "could not tell the mesh" means the write
|
|
// failed rather than that nobody has looked yet.
|
|
flush, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
return b.Conn.FlushWithContext(flush)
|
|
}
|