mesh-catalog/modules/mesh-conditions/cmd/mesh-conditions/hold.go

package main

// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12;
// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a
// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or
// about what it serves, is kept in the state
// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once,
// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's
// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery
// did not end. A finding about another module or about the machine itself is never held. A condition already open is
// not held: it is no news.

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/health"
)

// The module whose delivery holds what it serves beyond their own findings (Limits 12).
const runtimeModule = "node-tools"

// about is a node and a module a finding or a condition is about, for the hold.
type about struct{ Node, Module string }

// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its
// node, a seat of the same name as its module on a machine, and every module's tools on a machine whose tool
// runner is delivered. Never the machine itself.
func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about {
	var out []about
	parts := strings.Split(s.ID, ".")
	switch s.Scope {
	case health.ScopeModule:
		if len(parts) >= 2 {
			out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")})
		}
	case health.ScopeProvider:
		if len(parts) >= 2 {
			out = append(out, about{Node: parts[1], Module: parts[0]})
		}
	case health.ScopeSeat:
		// The module the placement says holds the seat on that node, or the seat's own name where none is read.
		if len(parts) >= 2 {
			node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".")
			module := k.holders[node+"."+seat]
			if module == "" {
				module = seat
			}
			out = append(out, about{Node: node, Module: module})
		}
	case health.ScopeMachine:
		// The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is
		// delivered.
		if source == "S11" || kind == "tools-silent" {
			out = append(out, about{Node: s.Machine, Module: runtimeModule})
			if s.Machine == "" && len(parts) > 0 {
				out[len(out)-1].Node = parts[0]
			}
		}
		return out
	}
	if s.Machine != "" && s.Scope != health.ScopeMachine {
		out = append(out, about{Node: s.Machine, Module: runtimeModule})
	}
	return out
}

func splitNodeModule(hk string) (string, string) {
	node, module, _ := strings.Cut(hk, ".")
	return node, module
}

// motion is the delivery of module on node, from its want and have.
func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) {
	w, wanted := k.wants[node+"."+module]
	h, has := k.haves[node+"."+module]
	return delivery.MotionOf(w, wanted, h, has, "", false), w
}

// holdFor says whether key is held now, and as what. Called with mu held.
func (k *Keeper) holdFor(key string, f *health.Finding, open bool) (held, bool) {
	if open || f == nil {
		return held{}, false
	}
	kind, source, subject := f.Kind, f.Source, f.Subject
	if _, raisedOut := k.outOfHold[key]; raisedOut {
		return held{}, false
	}
	for _, a := range k.aboutOf(kind, source, subject) {
		m, w := k.motion(a.Node, a.Module)
		if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP {
			continue
		}
		since := k.Now().UTC()
		if was, ok := k.held[key]; ok {
			since = was.Since
		}
		return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true
	}
	return held{}, false
}

// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once.
func (k *Keeper) keepHeld(h held) error {
	k.mu.Lock()
	_, was := k.held[h.Key]
	k.held[h.Key] = h
	bound := k.bound(h.Module)
	k.mu.Unlock()
	if was {
		return nil
	}
	if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) {
		return fmt.Errorf("keeping %s held: %w", h.Key, err)
	}
	b := boundKey(h.Node, h.Module)
	err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version,
		"bound-seconds": int(bound / time.Second)}, bound)
	if err != nil && !errors.Is(err, stdio.ErrChanged) {
		return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err)
	}
	k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound)
	return nil
}

// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state
// `bounds` is written with it as its own time to live, and its expiry is the bound passing.
func (k *Keeper) bound(string) time.Duration {
	if k.Bound > 0 {
		return k.Bound
	}
	return delivery.DefaultDeliveryBound
}

func boundKey(node, module string) string { return node + "." + module }

// dropHeld lets a held key go: raised, or gone with why said.
func (k *Keeper) dropHeld(key, why string) {
	k.mu.Lock()
	h, ok := k.held[key]
	delete(k.held, key)
	k.mu.Unlock()
	if !ok {
		return
	}
	if err := k.Store.Delete(HeldState, key); err != nil {
		k.Logf("the held %s could not be removed from held: %v", key, err)
	}
	if why != "" {
		k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why)
	}
}

// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or
// drops what it held.
func (k *Keeper) Want(key string, value []byte, deleted bool) error {
	node, module, err := delivery.ParseWantKey(key)
	if err != nil {
		return nil // not a want of a module on a node (a held fingerprint's key)
	}
	k.mu.Lock()
	k.init()
	if deleted {
		delete(k.wants, node+"."+module)
	} else if w, err := delivery.DecodeWant(value); err == nil {
		k.wants[node+"."+module] = w
	} else {
		k.Logf("the want %s is not readable: %v", key, err)
	}
	k.mu.Unlock()
	return k.judge(node, module, "")
}

// Have takes one change of a node's have.
func (k *Keeper) Have(key string, value []byte, deleted bool) error {
	node, module, isNode, err := delivery.ParseHaveKey(key)
	if err != nil || isNode {
		return nil
	}
	k.mu.Lock()
	k.init()
	if deleted {
		delete(k.haves, node+"."+module)
	} else if h, err := delivery.DecodeHave(value); err == nil {
		k.haves[node+"."+module] = h
	} else {
		k.Logf("the have %s is not readable: %v", key, err)
	}
	k.mu.Unlock()
	return k.judge(node, module, "")
}

// Placement takes one change of the placement: which module holds which seat on a node. The keys are read in either
// form, a node key and its module keys or the form before (delivery.PlacementKeys; research 072 row 37).
func (k *Keeper) Placement(key string, value []byte, deleted bool) {
	k.mu.Lock()
	defer k.mu.Unlock()
	k.init()
	node, err := k.placement.Put(key, value, 0, deleted)
	if node == "" {
		return
	}
	if err != nil {
		k.Logf("the placement of %s is not readable as it stands, so its seats' holders are kept as last read: %v", node, err)
	}
	for hk := range k.holders {
		if strings.HasPrefix(hk, node+".") {
			delete(k.holders, hk)
		}
	}
	p, _, ok := k.placement.Of(node)
	if !ok {
		return
	}
	for _, h := range p.Holds {
		k.holders[node+"."+h.Claim] = h.Module
	}
}

// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end.
func (k *Keeper) BoundPassed(key string) error {
	node, module := splitNodeModule(key)
	k.mu.Lock()
	k.init()
	_, w := k.motion(node, module)
	holding := false
	for _, h := range k.held {
		holding = holding || (h.Node == node && h.Module == module)
	}
	if !holding {
		k.mu.Unlock()
		return nil
	}
	k.passed[key] = w.FP
	k.mu.Unlock()
	return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node,
		k.bound(module)))
}

// judge raises or drops what node and module hold when their delivery ended, or with line past its bound.
func (k *Keeper) judge(node, module, line string) error {
	k.mu.Lock()
	m, w := k.motion(node, module)
	var keys []string
	for key, h := range k.held {
		if h.Node == node && h.Module == module {
			keys = append(keys, key)
		}
	}
	if line == "" && !m.Ended() && m.Delivering {
		k.mu.Unlock()
		return nil
	}
	if line == "" {
		line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node)
		if m.Failed {
			line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node)
		}
		delete(k.passed, node+"."+module)
	}
	for _, key := range keys {
		k.outOfHold[key] = line
	}
	k.mu.Unlock()
	sort.Strings(keys)
	var problems []string
	for _, key := range keys {
		k.dropHeld(key, "")
		if err := k.Settle(key); err != nil {
			problems = append(problems, err.Error())
		}
		k.mu.Lock()
		delete(k.outOfHold, key)
		k.mu.Unlock()
	}
	if len(problems) > 0 {
		return errors.New(strings.Join(problems, "; "))
	}
	return nil
}

func short(fp string) string {
	if _, rest, ok := strings.Cut(fp, ":"); ok {
		fp = rest
	}
	if len(fp) > 12 {
		return fp[:12]
	}
	return fp
}

// Held is every key held now, for the read tool.
func (k *Keeper) Held() []held {
	k.mu.Lock()
	defer k.mu.Unlock()
	out := make([]held, 0, len(k.held))
	for _, h := range k.held {
		out = append(out, h)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
	return out
}

// restoreHeld takes the state `held` as read at the start.
func (k *Keeper) restoreHeld(key string, value []byte) {
	var h held
	if json.Unmarshal(value, &h) != nil {
		return
	}
	k.mu.Lock()
	k.init()
	k.held[key] = h
	k.mu.Unlock()
}