A module's runtime waits for its broker credential and its account at a start, bounded, instead of exiting (issue 1496)
The node-engine starts a module's runtime before the provisioner's client fills its broker file, and the bus carries a new account only after it reloads its users. The runtime exited on both and was restarted until they were there (ace, 2026-10-26 21:19: mesh-build-node's and mesh-build-image's runtimes exited twice). It now waits up to five minutes from its start, said once: the file is read again each time its directory changes (the bundles' inotify watch, a 30 s backstop), and an authorization refusal is asked again with the backoff. Past the bound it gives up loudly; a file with no url, a pin mismatch and a bad URL stay final.
This commit is contained in:
@@ -12,7 +12,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
@@ -41,8 +43,9 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
cred := credential()
|
||||
conn := connectPatiently(cred)
|
||||
until := time.Now().Add(startWait)
|
||||
cred := credential(until)
|
||||
conn := connectPatiently(cred, until)
|
||||
|
||||
served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module)
|
||||
if err != nil {
|
||||
@@ -97,48 +100,148 @@ func main() {
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// credential reads the credential the mesh delivered; a missing or unreadable one is a fault of
|
||||
// configuration, said and final.
|
||||
func credential() bus.Credential {
|
||||
// startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus
|
||||
// to accept the account it names (issue 1496): the node-engine starts a module's runtime before the provisioner's client
|
||||
// fills the credential, and the bus carries a new account only after it reloads its users. Past it the runtime gives up,
|
||||
// loudly; inside it the wait is said once.
|
||||
var startWait = 5 * time.Minute
|
||||
|
||||
// say is a line in the runtime's journal.
|
||||
func say(format string, args ...any) { fmt.Fprintf(os.Stderr, "node-tools: "+format+"\n", args...) }
|
||||
|
||||
// credential reads the credential the mesh delivered, waiting for it within the start's bound (issue 1496); one that can
|
||||
// never be a credential (no url) is a fault of configuration, said and final.
|
||||
func credential(until time.Time) bus.Credential {
|
||||
file := os.Getenv("MESH_BROKER_FILE")
|
||||
if file == "" {
|
||||
if url := os.Getenv("MESH_BROKER_URL"); url != "" {
|
||||
return bus.Credential{URL: url, Module: runtimeModule}
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "node-tools: set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
say("set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
os.Exit(1)
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
cred, err := awaitCredential(file, until, say)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cred.URL == "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s carries no url — it is not a broker credential\n", file)
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup —
|
||||
// and gives up at once on what waiting cannot fix (issue 058).
|
||||
func connectPatiently(cred bus.Credential) *bus.Conn {
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := bus.Connect(cred)
|
||||
if err == nil {
|
||||
return conn
|
||||
// errNotCredential is a file that is there and readable and still no broker credential: waiting will not fix it.
|
||||
var errNotCredential = errors.New("it carries no url — it is not a broker credential")
|
||||
|
||||
// readCredential reads the credential at file. A file not there yet, empty, or not yet whole JSON is one the node-engine
|
||||
// has still to write: the error says so and waiting may fix it. errNotCredential is final.
|
||||
func readCredential(file string) (bus.Credential, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return bus.Credential{}, errors.New("the file is empty")
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if cred.URL == "" {
|
||||
return bus.Credential{}, errNotCredential
|
||||
}
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
// awaitCredential is the credential at file, read at once and, while it is not there or not whole, again each time its
|
||||
// directory changes (runtime.WatchFile, inotify), until the time until. The wait is said once; at until it ends with an
|
||||
// error naming the file, how long it waited and what the last read found. A file that is not a credential ends it at once.
|
||||
func awaitCredential(file string, until time.Time, say func(string, ...any)) (bus.Credential, error) {
|
||||
cred, err := readCredential(file)
|
||||
if err == nil {
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
began := time.Now()
|
||||
say("waits for its broker credential at %s (%v): the node-engine writes it once the provisioner's client fills it; "+
|
||||
"it gives up at %s", file, err, until.Format("15:04:05"))
|
||||
changed := make(chan struct{}, 1)
|
||||
stop, werr := runtime.WatchFile(file, func() {
|
||||
select {
|
||||
case changed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
if fatal := bus.Fatal(err); fatal != "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s — waiting will not fix this; giving up\n", fatal)
|
||||
os.Exit(1)
|
||||
}, func(format string, args ...any) { say(format, args...) })
|
||||
if werr != nil {
|
||||
say("cannot watch %s (%v): it is read again every %s instead", file, werr, rereadEvery)
|
||||
} else {
|
||||
defer stop()
|
||||
}
|
||||
for {
|
||||
left := time.Until(until)
|
||||
if left <= 0 {
|
||||
return bus.Credential{}, fmt.Errorf("cannot read the broker credential at %s: waited %s for it, and the last read "+
|
||||
"found: %v", file, time.Since(began).Round(time.Second), err)
|
||||
}
|
||||
// The watch wakes the read; the clock is only the backstop for a watch that could not be made or missed a change.
|
||||
select {
|
||||
case <-changed:
|
||||
case <-time.After(min(left, rereadEvery)):
|
||||
}
|
||||
if cred, err = readCredential(file); err == nil {
|
||||
say("read its broker credential at %s after %s", file, time.Since(began).Round(time.Second))
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rereadEvery is the backstop of the wait for the credential, beside the watch of its directory.
|
||||
var rereadEvery = 30 * time.Second
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup — and, until the start's
|
||||
// bound, while the bus refuses the account: the bus carries a new account only after it reloads its users, which may come
|
||||
// after the credential is on the machine (issue 1496, issue 1423). What waiting cannot fix (a pin that does not match, an
|
||||
// address that is no URL) gives up at once (issue 058), and so does a refusal past the bound.
|
||||
func connectPatiently(cred bus.Credential, until time.Time) *bus.Conn {
|
||||
conn, err := connectWithin(cred, until, bus.Connect, time.Sleep, say)
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// connectWithin is connectPatiently with its dial, its sleep and its words given, as a test gives them.
|
||||
func connectWithin(cred bus.Credential, until time.Time, dial func(bus.Credential) (*bus.Conn, error), sleep func(time.Duration),
|
||||
say func(string, ...any)) (*bus.Conn, error) {
|
||||
refusedSaid := false
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := dial(cred)
|
||||
if err == nil {
|
||||
if refusedSaid {
|
||||
say("the bus accepts the account now")
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
wait := delay + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
fmt.Fprintf(os.Stderr, "node-tools: the broker is not reachable yet (%v); retrying in %ds\n", err, int(wait.Round(time.Second)/time.Second))
|
||||
time.Sleep(wait)
|
||||
switch fatal := bus.Fatal(err); {
|
||||
case bus.Refused(err) && time.Now().Add(wait).Before(until):
|
||||
if !refusedSaid {
|
||||
say("the bus refuses this account (%v): it may not carry it yet, since it loads a new account by a reload; "+
|
||||
"retrying until %s", err, until.Format("15:04:05"))
|
||||
refusedSaid = true
|
||||
}
|
||||
case bus.Refused(err) && refusedSaid:
|
||||
return nil, fmt.Errorf("the bus still refuses this account (%v) at the end of the start's bound, %s; giving up", err,
|
||||
until.Format("15:04:05"))
|
||||
case fatal != "":
|
||||
return nil, fmt.Errorf("%s — waiting will not fix this; giving up", fatal)
|
||||
default:
|
||||
say("the broker is not reachable yet (%v); retrying in %ds", err, int(wait.Round(time.Second)/time.Second))
|
||||
}
|
||||
sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user