package build import ( "context" "encoding/json" "errors" "fmt" "slices" "strings" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "mesh-delivery-builder/internal/versions" ) // **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673 // §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build // seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`. // Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its // newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event // starts nothing and asks the operator once. // MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked. const MostEventBuilds = 3 // Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the // key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module // has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it // stands and its revision. func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) { b.init() module, fp, err := builds.ParseBuilderBuildKey(key) if err != nil { return false, false, cur, 0, err } value, rev, found, err := b.Claimer.Store.Get(ctx, key) if err != nil || !found { return false, false, cur, 0, err } if cur, err = builds.DecodeBuilderBuild(value); err != nil { return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err) } if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) { return false, false, cur, rev, nil } if cur.EventBuilds >= MostEventBuilds { return false, true, cur, rev, nil } b.mu.Lock() m, known := b.known[module] if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil { b.mu.Unlock() return false, false, cur, rev, nil } b.inflight[key], b.events[key] = true, true b.mu.Unlock() b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds) b.startOnce(ctx, key, module, m) return true, false, cur, rev, nil } // Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake). type Retaker struct { Builder *Builder // Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at. Acted versions.ReserveStore // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) // Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask). Ask func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error Logf func(string, ...any) } func (r *Retaker) logf(f string, a ...any) { if r.Logf != nil { r.Logf(f, a...) } } // retakeKey is the acted key of a subject's sequence, apart from the publish again's. func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) } // askedKey is the acted key of the revision of a build key the operator was asked at. func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) } // Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on // it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again. func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") if !running && !changed { return false, nil } key := retakeKey(subject) raw, rev, found, err := r.Acted.Get(ctx, key) if err != nil { return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) } if found { var a versions.Acted if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { return true, nil } } var want func(seat, node string) bool if running { want, err = r.runningWant(ctx, data) } else { want, err = r.returnedWant(data) } if err == nil && want != nil { err = r.retakeAll(ctx, want) } if err != nil { return true, err } val, _ := json.Marshal(versions.Acted{Sequence: seq}) if found { _, err = r.Acted.Update(ctx, key, val, rev) } else { _, err = r.Acted.Create(ctx, key, val) } if err != nil && !errors.Is(err, versions.ErrConflict) { return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) } return true, nil } // runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node. func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) { var ev nodeengine.Running if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { r.logf("a running that does not read is not acted on: %v", err) return nil, nil } if ev.Stopped || ev.Fingerprint == "" { return nil, nil } e, err := r.Entry(ctx, ev.Fingerprint) if err != nil { return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) } var seats []string for _, s := range e.Serves { if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) { seats = append(seats, seat) } } if len(seats) == 0 { return nil, nil } return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil } // returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names. func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) { var ev nodeengine.ManifestReadsChanged if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) return nil, nil } if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { return nil, nil } return func(_, node string) bool { return node == ev.Node }, nil } // retakeAll builds again every key failed on a transport fault whose seat and node want says passed. func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error { keys, err := r.Builder.Claimer.Store.Keys(ctx) if err != nil { return fmt.Errorf("the build keys cannot be listed: %w", err) } for _, key := range keys { raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key) if err != nil || !found { continue } v, err := builds.DecodeBuilderBuild(raw) if err != nil || v.State != builds.BuildFailed { continue } _, seat, node, _, ok := builds.TransportFaultOf(v.Error) if !ok || !want(seat, node) { continue } started, ask, cur, rev, err := r.Builder.Retake(ctx, key) if err != nil { return err } if started || !ask || r.Ask == nil { continue } if err := r.askOnce(ctx, key, cur, rev); err != nil { return err } } return nil } // askOnce asks the operator about key once per revision of the key. func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error { ak := askedKey(key) raw, arev, found, err := r.Acted.Get(ctx, ak) if err != nil { return err } val := []byte(fmt.Sprint(rev)) if found && string(raw) == string(val) { return nil } if err := r.Ask(ctx, key, cur, rev); err != nil { return fmt.Errorf("the operator cannot be asked about %s: %w", key, err) } if found { _, err = r.Acted.Update(ctx, ak, val, arev) } else { _, err = r.Acted.Create(ctx, ak, val) } if err != nil && !errors.Is(err, versions.ErrConflict) { return err } return nil }