jschoubben
50cca4309f
The builder's asker on the seat operator-channel (level approve, about the key, state asked), the teller's three bus adapters (publish, the router's record through the seat's holder on the seat list, decided heard through the hub's events handler); the manifest uses the seat and consumes decided. Builds: design 47 G §568, A-74 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
227 lines
8.3 KiB
Go
227 lines
8.3 KiB
Go
package build
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"mesh-delivery-builder/internal/versions"
|
|
)
|
|
|
|
// **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673
|
|
// §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build
|
|
// seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`.
|
|
// Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its
|
|
// newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event
|
|
// starts nothing and asks the operator once.
|
|
|
|
// MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked.
|
|
const MostEventBuilds = 3
|
|
|
|
// Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the
|
|
// key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module
|
|
// has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it
|
|
// stands and its revision.
|
|
func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) {
|
|
b.init()
|
|
module, fp, err := builds.ParseBuilderBuildKey(key)
|
|
if err != nil {
|
|
return false, false, cur, 0, err
|
|
}
|
|
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
|
|
if err != nil || !found {
|
|
return false, false, cur, 0, err
|
|
}
|
|
if cur, err = builds.DecodeBuilderBuild(value); err != nil {
|
|
return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
|
}
|
|
if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) {
|
|
return false, false, cur, rev, nil
|
|
}
|
|
if cur.EventBuilds >= MostEventBuilds {
|
|
return false, true, cur, rev, nil
|
|
}
|
|
b.mu.Lock()
|
|
m, known := b.known[module]
|
|
if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil {
|
|
b.mu.Unlock()
|
|
return false, false, cur, rev, nil
|
|
}
|
|
b.inflight[key], b.events[key] = true, true
|
|
b.mu.Unlock()
|
|
b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds)
|
|
b.startOnce(ctx, key, module, m)
|
|
return true, false, cur, rev, nil
|
|
}
|
|
|
|
// Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake).
|
|
type Retaker struct {
|
|
Builder *Builder
|
|
// Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at.
|
|
Acted versions.ReserveStore
|
|
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
|
|
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
|
|
// Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask).
|
|
Ask func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error
|
|
Logf func(string, ...any)
|
|
}
|
|
|
|
func (r *Retaker) logf(f string, a ...any) {
|
|
if r.Logf != nil {
|
|
r.Logf(f, a...)
|
|
}
|
|
}
|
|
|
|
// retakeKey is the acted key of a subject's sequence, apart from the publish again's.
|
|
func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) }
|
|
|
|
// askedKey is the acted key of the revision of a build key the operator was asked at.
|
|
func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) }
|
|
|
|
// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on
|
|
// it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again.
|
|
func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) {
|
|
running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*"))
|
|
changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".")
|
|
if !running && !changed {
|
|
return false, nil
|
|
}
|
|
key := retakeKey(subject)
|
|
raw, rev, found, err := r.Acted.Get(ctx, key)
|
|
if err != nil {
|
|
return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err)
|
|
}
|
|
if found {
|
|
var a versions.Acted
|
|
if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq {
|
|
return true, nil
|
|
}
|
|
}
|
|
var want func(seat, node string) bool
|
|
if running {
|
|
want, err = r.runningWant(ctx, data)
|
|
} else {
|
|
want, err = r.returnedWant(data)
|
|
}
|
|
if err == nil && want != nil {
|
|
err = r.retakeAll(ctx, want)
|
|
}
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
val, _ := json.Marshal(versions.Acted{Sequence: seq})
|
|
if found {
|
|
_, err = r.Acted.Update(ctx, key, val, rev)
|
|
} else {
|
|
_, err = r.Acted.Create(ctx, key, val)
|
|
}
|
|
if err != nil && !errors.Is(err, versions.ErrConflict) {
|
|
return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node.
|
|
func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) {
|
|
var ev nodeengine.Running
|
|
if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
|
|
r.logf("a running that does not read is not acted on: %v", err)
|
|
return nil, nil
|
|
}
|
|
if ev.Stopped || ev.Fingerprint == "" {
|
|
return nil, nil
|
|
}
|
|
e, err := r.Entry(ctx, ev.Fingerprint)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
|
|
}
|
|
var seats []string
|
|
for _, s := range e.Serves {
|
|
if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) {
|
|
seats = append(seats, seat)
|
|
}
|
|
}
|
|
if len(seats) == 0 {
|
|
return nil, nil
|
|
}
|
|
return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil
|
|
}
|
|
|
|
// returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names.
|
|
func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) {
|
|
var ev nodeengine.ManifestReadsChanged
|
|
if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
|
|
r.logf("a manifest-reads-changed that does not read is not acted on: %v", err)
|
|
return nil, nil
|
|
}
|
|
if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
|
|
return nil, nil
|
|
}
|
|
return func(_, node string) bool { return node == ev.Node }, nil
|
|
}
|
|
|
|
// retakeAll builds again every key failed on a transport fault whose seat and node want says passed.
|
|
func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error {
|
|
keys, err := r.Builder.Claimer.Store.Keys(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the build keys cannot be listed: %w", err)
|
|
}
|
|
for _, key := range keys {
|
|
raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key)
|
|
if err != nil || !found {
|
|
continue
|
|
}
|
|
v, err := builds.DecodeBuilderBuild(raw)
|
|
if err != nil || v.State != builds.BuildFailed {
|
|
continue
|
|
}
|
|
_, seat, node, _, ok := builds.TransportFaultOf(v.Error)
|
|
if !ok || !want(seat, node) {
|
|
continue
|
|
}
|
|
started, ask, cur, rev, err := r.Builder.Retake(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if started || !ask || r.Ask == nil {
|
|
continue
|
|
}
|
|
if err := r.askOnce(ctx, key, cur, rev); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// askOnce asks the operator about key once per revision of the key.
|
|
func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error {
|
|
ak := askedKey(key)
|
|
raw, arev, found, err := r.Acted.Get(ctx, ak)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
val := []byte(fmt.Sprint(rev))
|
|
if found && string(raw) == string(val) {
|
|
return nil
|
|
}
|
|
if err := r.Ask(ctx, key, cur, rev); err != nil {
|
|
return fmt.Errorf("the operator cannot be asked about %s: %w", key, err)
|
|
}
|
|
if found {
|
|
_, err = r.Acted.Update(ctx, ak, val, arev)
|
|
} else {
|
|
_, err = r.Acted.Create(ctx, ak, val)
|
|
}
|
|
if err != nil && !errors.Is(err, versions.ErrConflict) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|