mesh-delivery/modules/mesh-delivery-watchman/internal/watch/mirror.go

package watch

import (
	"strings"
	"time"

	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)

// A build ask taken by a node is a move (novox/hq ADR 0525 §4, issues 1887 A8b, 1908, 1909): the node writes the ask's step
// key `<node>.<ask id>` running in the build module's steps (StepsBucket) when it takes the ask, and writes it passed,
// failed or refused when the ask ends. The pulse of the move is the pulse of the bucket named as the seat's stream
// (buildstep.AskStream), under the ask's id, and it begins when the node started the step.
//
// **An ask still waiting in the stream is a backlog, not a stall** (issue 1908): the rebuild after the switch puts about 170
// asks on a few nodes' slots, and most wait for hours with every worker healthy. No pulse is made for a waiting ask, so none
// can expire into a false stuck row; the view shows the backlog and the oldest waiting ask's age (delivery view, A8a). What
// is read of the steps at the watchman's start is mirrored the same way, so an ask running when the watchman was down is
// pulsed again (issue 1909).

// Step is a change of a key of the steps bucket of language's build module, read from the watch (at the start the keys that
// stand, then each change). A running step makes the pulse of its ask, from the step's own start; a step that ended, or a
// key deleted, ends it. The node's `paused.<node>` key, a key that is no step key and a value that cannot be read are left.
func (w *Watchman) Step(language, key string, value []byte, op Op) {
	bucket, err := buildstep.AskStream(language)
	if err != nil {
		w.say("no pulse for the step %s: %v", key, err)
		return
	}
	node, id, err := buildstep.ParseStepKey(key)
	if err != nil {
		return
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	switch op {
	case Expired:
		return
	case Delete:
		w.moved(bucket, id)
		return
	}
	w.stands(bucket, id)
	step, err := buildstep.DecodeStep(value)
	if err != nil {
		w.say("the step %s cannot be read, and is left: %v", key, err)
		return
	}
	if step.State != buildstep.StepRunning {
		w.moved(bucket, id)
		return
	}
	since := step.Started
	if since.IsZero() {
		since = w.now()
	}
	w.mirrorWithin(w.stepBound, bucket, id, describe("the ask %s to end on %s", id, node), node, since, 0)
}

// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
// and the node's own key) are left.

// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
// change), at the time the entry was written.
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
	node, module, err := meshwantsclient.ParseWantKey(key)
	if err != nil {
		return
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	switch op {
	case Expired:
		return
	case Delete:
		delete(w.wants, key)
		delete(w.held, key)
		w.moved(AssignerWantsBucket, key)
		return
	}
	w.stands(AssignerWantsBucket, key)
	want, err := meshwantsclient.DecodeWant(value)
	if err != nil {
		w.say("the want %s cannot be read, and is left: %v", key, err)
		return
	}
	if want.Removed {
		delete(w.wants, key)
		delete(w.held, key)
		w.moved(AssignerWantsBucket, key)
		return
	}
	w.wants[key] = want.Fp
	if want.Newest != "" || want.Rule != 0 {
		// A want held by a rule (ADR 0603 §8) waits on the rule, not on the node: its wait is the hold's, from the time the
		// assigner wrote it, and the node running what the want names does not end it.
		if at.IsZero() {
			at = w.now()
		}
		w.held[key] = true
		w.mirrorKept(AssignerWantsBucket, key,
			describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.Fp), want.Rule, w.stepBound, want.Words),
			"held|"+want.Fp+"|"+want.Newest, at)
		return
	}
	if w.held[key] {
		delete(w.held, key)
		if pk, err := meshwatchdogsclient.PulseKey(AssignerWantsBucket, key); err == nil {
			delete(w.starts, pk)
		}
		w.stuckMoved(AssignerWantsBucket, key) // the hold ended: its finding is cleared
	}
	if w.have(node, module) == want.Fp {
		w.moved(AssignerWantsBucket, key)
		return
	}
	if at.IsZero() {
		at = w.now()
	}
	w.mirrorWithin(w.stepBound, AssignerWantsBucket, key,
		describe("the node agent of %s to run %s at %s", node, module, shortFP(want.Fp)), want.Fp, at, 0)
}

// A build key queued behind a package is a wait of the builder's (ADR 0603 §8, ADR 0500 clause 6): its Waits names the package
// version the registry does not yet hold, and the key is not claimed until it is published. Its pulse is the pulse of the
// builder's builds bucket under the key, from the time the key was written, bounded as a step is (StepBound); the key
// claimed, done, failed or deleted ends it.

// Build is a change of a key of the builder's builds bucket, read from the watch (at the start the keys that stand, then
// each change), at the time the entry was written.
func (w *Watchman) Build(key string, value []byte, op Op, at time.Time) {
	module, fp, err := builds.ParseBuilderBuildKey(key)
	if err != nil {
		return
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	if op == Expired {
		return
	}
	if op == Delete {
		w.moved(BuilderBuildsBucket, key)
		return
	}
	w.stands(BuilderBuildsBucket, key)
	b, err := builds.DecodeBuilderBuild(value)
	if err != nil {
		w.say("the build %s cannot be read, and is left: %v", key, err)
		return
	}
	if b.State != builds.BuildQueued || b.Waits == "" {
		w.moved(BuilderBuildsBucket, key)
		return
	}
	if at.IsZero() {
		at = w.now()
	}
	w.mirrorKept(BuilderBuildsBucket, key,
		describe("the package %s to be published before the build of %s at %s, bound %s", b.Waits, module, shortFP(fp), w.stepBound), b.Waits, at)
}

// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the
// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left.
func (w *Watchman) Have(key string, value []byte, op Op) {
	node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
	if err != nil {
		return
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	if isNode {
		// The node's own record: its tunnel key, whose peers' sight of it is network-unreachable.
		var record *nodeengine.NodeRecord
		if op == Put {
			if hn, err := nodeengineclient.DecodeHaveNode(value); err == nil {
				record = hn.Record
			}
		}
		w.tunnelKey(node, record)
		return
	}
	switch op {
	case Expired:
		return
	case Delete:
		delete(w.haves, key)
		w.haveFindings(key, node, module, nil)
		return
	}
	have, err := nodeengineclient.DecodeHave(value)
	if err != nil {
		w.say("the have %s cannot be read, and is left: %v", key, err)
		return
	}
	w.haves[key] = have.Fp
	w.haveFindings(key, node, module, &have)
	wk, err := meshwantsclient.WantKey(node, module)
	if err != nil {
		return
	}
	if fp, ok := w.wants[wk]; ok && fp == have.Fp {
		w.moved(AssignerWantsBucket, wk)
	}
}

func (w *Watchman) have(node, module string) string {
	key, err := nodeengineclient.HaveKey(node, module)
	if err != nil {
		return ""
	}
	return w.haves[key]
}

func shortFP(fp string) string {
	hex := strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
	if len(hex) > 12 {
		hex = hex[:12]
	}
	return hex
}