mesh-delivery/modules/mesh-delivery-builder/internal/build/retake.go

package build

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"mesh-delivery-builder/internal/versions"
)

// **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673
// §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build
// seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`.
// Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its
// newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event
// starts nothing and asks the operator once.

// MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked.
const MostEventBuilds = 3

// Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the
// key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module
// has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it
// stands and its revision.
func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) {
	b.init()
	module, fp, err := builds.ParseBuilderBuildKey(key)
	if err != nil {
		return false, false, cur, 0, err
	}
	value, rev, found, err := b.Claimer.Store.Get(ctx, key)
	if err != nil || !found {
		return false, false, cur, 0, err
	}
	if cur, err = builds.DecodeBuilderBuild(value); err != nil {
		return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err)
	}
	if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) {
		return false, false, cur, rev, nil
	}
	if cur.EventBuilds >= MostEventBuilds {
		return false, true, cur, rev, nil
	}
	b.mu.Lock()
	m, known := b.known[module]
	if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil {
		b.mu.Unlock()
		return false, false, cur, rev, nil
	}
	b.inflight[key], b.events[key] = true, true
	b.mu.Unlock()
	b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds)
	b.startOnce(ctx, key, module, m)
	return true, false, cur, rev, nil
}

// Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake).
type Retaker struct {
	Builder *Builder
	// Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at.
	Acted versions.ReserveStore
	// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
	Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
	// Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask).
	Ask  func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error
	Logf func(string, ...any)
}

func (r *Retaker) logf(f string, a ...any) {
	if r.Logf != nil {
		r.Logf(f, a...)
	}
}

// retakeKey is the acted key of a subject's sequence, apart from the publish again's.
func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) }

// askedKey is the acted key of the revision of a build key the operator was asked at.
func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) }

// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on
// it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again.
func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) {
	running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*"))
	changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".")
	if !running && !changed {
		return false, nil
	}
	key := retakeKey(subject)
	raw, rev, found, err := r.Acted.Get(ctx, key)
	if err != nil {
		return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err)
	}
	if found {
		var a versions.Acted
		if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq {
			return true, nil
		}
	}
	var want func(seat, node string) bool
	if running {
		want, err = r.runningWant(ctx, data)
	} else {
		want, err = r.returnedWant(data)
	}
	if err == nil && want != nil {
		err = r.retakeAll(ctx, want)
	}
	if err != nil {
		return true, err
	}
	val, _ := json.Marshal(versions.Acted{Sequence: seq})
	if found {
		_, err = r.Acted.Update(ctx, key, val, rev)
	} else {
		_, err = r.Acted.Create(ctx, key, val)
	}
	if err != nil && !errors.Is(err, versions.ErrConflict) {
		return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err)
	}
	return true, nil
}

// runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node.
func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) {
	var ev nodeengine.Running
	if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
		r.logf("a running that does not read is not acted on: %v", err)
		return nil, nil
	}
	if ev.Stopped || ev.Fingerprint == "" {
		return nil, nil
	}
	e, err := r.Entry(ctx, ev.Fingerprint)
	if err != nil {
		return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
	}
	var seats []string
	for _, s := range e.Serves {
		if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) {
			seats = append(seats, seat)
		}
	}
	if len(seats) == 0 {
		return nil, nil
	}
	return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil
}

// returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names.
func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) {
	var ev nodeengine.ManifestReadsChanged
	if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
		r.logf("a manifest-reads-changed that does not read is not acted on: %v", err)
		return nil, nil
	}
	if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
		return nil, nil
	}
	return func(_, node string) bool { return node == ev.Node }, nil
}

// retakeAll builds again every key failed on a transport fault whose seat and node want says passed.
func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error {
	keys, err := r.Builder.Claimer.Store.Keys(ctx)
	if err != nil {
		return fmt.Errorf("the build keys cannot be listed: %w", err)
	}
	for _, key := range keys {
		raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key)
		if err != nil || !found {
			continue
		}
		v, err := builds.DecodeBuilderBuild(raw)
		if err != nil || v.State != builds.BuildFailed {
			continue
		}
		_, seat, node, _, ok := builds.TransportFaultOf(v.Error)
		if !ok || !want(seat, node) {
			continue
		}
		started, ask, cur, rev, err := r.Builder.Retake(ctx, key)
		if err != nil {
			return err
		}
		if started || !ask || r.Ask == nil {
			continue
		}
		if err := r.askOnce(ctx, key, cur, rev); err != nil {
			return err
		}
	}
	return nil
}

// askOnce asks the operator about key once per revision of the key.
func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error {
	ak := askedKey(key)
	raw, arev, found, err := r.Acted.Get(ctx, ak)
	if err != nil {
		return err
	}
	val := []byte(fmt.Sprint(rev))
	if found && string(raw) == string(val) {
		return nil
	}
	if err := r.Ask(ctx, key, cur, rev); err != nil {
		return fmt.Errorf("the operator cannot be asked about %s: %w", key, err)
	}
	if found {
		_, err = r.Acted.Update(ctx, ak, val, arev)
	} else {
		_, err = r.Acted.Create(ctx, ak, val)
	}
	if err != nil && !errors.Is(err, versions.ErrConflict) {
		return err
	}
	return nil
}