jochen
0c405b70cc
The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
126 lines
4.8 KiB
Go
126 lines
4.8 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
)
|
|
|
|
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
|
|
// node tools it placed on this machine, by what the bus says about them rather than by what they
|
|
// say about themselves.
|
|
//
|
|
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
|
|
// by JetStream's direct get — one subject, no stream information, nothing written; and this
|
|
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
|
|
// so only this machine's runtime can answer it.
|
|
|
|
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
|
|
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
|
|
// counts none of it against the build's bound.
|
|
var ErrCannotAsk = errors.New("this host cannot ask the bus")
|
|
|
|
// ErrNoAnswer is a question the bus delivered and nobody answered in time: the build being judged is
|
|
// not there to answer. Counted against its bound.
|
|
var ErrNoAnswer = errors.New("nothing answered")
|
|
|
|
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
|
|
type Asker interface {
|
|
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
|
|
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
|
|
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
|
|
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
|
|
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
|
|
Ping(ctx context.Context, service, id string) error
|
|
}
|
|
|
|
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
|
|
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
|
|
func DirectGetSubject(bucket, key string) string {
|
|
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
|
|
}
|
|
|
|
// PingSubject is the services protocol's PING of one instance of one service.
|
|
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
|
|
|
|
// Asker is the link open now, when there is one and it can ask.
|
|
func (q *Queue) Asker() Asker {
|
|
q.init()
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
if a, ok := q.bus.(Asker); ok {
|
|
return a
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
|
|
subject := DirectGetSubject(bucket, key)
|
|
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
|
|
switch {
|
|
case errors.Is(err, nats.ErrNoResponders):
|
|
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
|
|
case err != nil:
|
|
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject))
|
|
}
|
|
if msg.Header != nil {
|
|
switch status := msg.Header.Get("Status"); status {
|
|
case "":
|
|
case "404":
|
|
return nil, false, nil
|
|
default:
|
|
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
|
|
bucket, status, msg.Header.Get("Description"))
|
|
}
|
|
switch msg.Header.Get("KV-Operation") {
|
|
case "DEL", "PURGE":
|
|
return nil, false, nil
|
|
}
|
|
}
|
|
if len(msg.Data) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
return msg.Data, true, nil
|
|
}
|
|
|
|
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
|
|
subject := PingSubject(service, id)
|
|
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
|
|
switch {
|
|
case errors.Is(err, nats.ErrNoResponders):
|
|
// The bus delivered the question and nothing subscribes to it: the runtime is not on the bus.
|
|
return fmt.Errorf("%w: no %s on this machine is on the bus", ErrNoAnswer, service)
|
|
case err != nil:
|
|
if refused := l.refusal(subject); refused != "" {
|
|
return fmt.Errorf("%w: asking %s: %v%s", ErrCannotAsk, subject, err, refused)
|
|
}
|
|
return fmt.Errorf("%w: %s did not answer: %v", ErrNoAnswer, subject, err)
|
|
}
|
|
var ping struct {
|
|
Name string `json:"name"`
|
|
ID string `json:"id"`
|
|
}
|
|
if err := json.Unmarshal(msg.Data, &ping); err != nil || ping.Name != service || ping.ID != id {
|
|
return fmt.Errorf("%w: what answered %s is not %s %s", ErrNoAnswer, subject, service, id)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// refusal is the bus's last word on this connection when it refused a publish to subject, said as
|
|
// the end of an error; empty when it did not.
|
|
func (l *natsLink) refusal(subject string) string {
|
|
last := l.conn.LastError()
|
|
if last == nil {
|
|
return ""
|
|
}
|
|
words := strings.ToLower(last.Error())
|
|
if strings.Contains(words, "permissions violation") && strings.Contains(last.Error(), subject) {
|
|
return " — the bus refused it: this host's grant does not name " + subject
|
|
}
|
|
return ""
|
|
}
|