mesh-catalog/modules/mesh-build-go/buildstep/conformance/leftrunning.go

package conformance

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// **A step left running by a previous process** (novox/hq issue 1890, note 21): when a delivery replaced mesh-build-go
// on a node while a check ask ran, the new process's start sweep let the ask's key stand as running, and it held one
// of the node's build slots until a hand end. Every build module runs this case (LeftRunning) through the start sweep
// its main runs: the hand-started steps a previous process left are ended at once and loud, its asks' keys deleted (the
// stream redelivers the ask, novox/hq ADR 0525 §6, and its asker keeps waiting for that), what this process runs kept. A
// catalogue check (mesh-catalog checks/failurekit) refuses a build module whose tests do not call LeftRunning.

// Sweep is a build module's own start sweep, as its main runs it once at its start: it ends node's steps in scan that
// no process of the module runs any more (buildstep.EndStaleAtStart, here with the runtime already serving), before
// being the start of the process, saying each end with logf.
type Sweep func(ctx context.Context, scan buildstep.Scan, node string, before time.Time, logf func(string, ...any))

// LeftNode is the node the case's steps are of.
const LeftNode = "kitnode"

// leftCase is one step in the steps state before the sweep, and whether the sweep must end it.
type leftCase struct {
	id, tool, owner, state string
	node                   string
	age                    time.Duration // started this long before the process began (negative: after it)
	ended                  bool
	deleted                bool // a gone process's ask: its key is deleted, not ended
}

// gone is a previous process's owner on the same node.
var gone = "b00t-of-the-kit.4242.mvp0000"

func leftCases() []leftCase {
	return []leftCase{
		{id: "mvpwlkzi7e6ca8", tool: "check ask", owner: gone, age: 5 * time.Minute, deleted: true}, // note 21's ghost
		{id: "mvpwlkzi7e6ca9", tool: "delivery ask", owner: gone, age: -time.Second, deleted: true},
		{id: "gonehand1", tool: "build_x_test", owner: gone, age: time.Minute, ended: true},
		{id: "oldhand1", tool: "build_x_fetch", age: time.Hour, ended: true},                // no owner: an older build's
		{id: "mvpwlkzi7e6caa", tool: "check ask", age: time.Hour, ended: true},              // no owner, past the bound
		{id: "mvpwlkzi7e6cab", tool: "check ask", age: 5 * time.Minute},                     // no owner, the stream's still
		{id: "mvpwlkzi7e6cac", tool: "check ask", owner: buildstep.Owner, age: time.Minute}, // this process's
		{id: "minehand1", tool: "build_x_test", owner: buildstep.Owner, age: -time.Second},
		{id: "donehand1", tool: "build_x_test", owner: gone, age: time.Hour, state: delivery.StepPassed},
		{id: "otherhand1", tool: "build_x_test", owner: gone, age: time.Hour, node: "kitother"},
	}
}

// LeftRunning runs the case through sweep and fails the test for each step the sweep ended or kept wrongly, or ended
// without saying it.
func LeftRunning(t *testing.T, sweep Sweep) {
	t.Helper()
	for _, problem := range JudgeLeftRunning(sweep) {
		t.Error(problem)
	}
}

// JudgeLeftRunning says what is wrong with what sweep did to the case; nothing when it keeps the rule.
func JudgeLeftRunning(sweep Sweep) []string {
	st := &steps{vals: map[string][]byte{}, revs: map[string]uint64{}}
	before := time.Now()
	for _, c := range leftCases() {
		node, state := c.node, c.state
		if node == "" {
			node = LeftNode
		}
		if state == "" {
			state = delivery.StepRunning
		}
		key, _ := delivery.StepKey(node, c.id)
		raw, err := delivery.EncodeStep(delivery.Step{ID: c.id, Tool: c.tool, Node: node, State: state, Started: before.Add(-c.age).UTC(), Owner: c.owner})
		if err != nil {
			return []string{"the kit's step " + c.id + ": " + err.Error()}
		}
		st.set(key, raw)
	}
	served := make(chan struct{})
	close(served)
	was := buildstep.StaleServed
	buildstep.StaleServed = func() <-chan struct{} { return served }
	defer func() { buildstep.StaleServed = was }()
	var mu sync.Mutex
	var said []string
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	sweep(ctx, st.scan(), LeftNode, before, func(f string, a ...any) {
		mu.Lock()
		defer mu.Unlock()
		said = append(said, fmt.Sprintf(f, a...))
	})
	mu.Lock()
	log := strings.Join(said, "\n")
	mu.Unlock()
	var problems []string
	for _, c := range leftCases() {
		node := c.node
		if node == "" {
			node = LeftNode
		}
		key, _ := delivery.StepKey(node, c.id)
		raw := st.get(key)
		if c.deleted {
			if raw != nil {
				problems = append(problems, fmt.Sprintf("left running: the ask %s of the gone owner %q is still held after the sweep (%s); "+
					"its key is deleted, never ended: the stream redelivers it (ADR 0525 §6)", key, c.owner, raw))
			}
			if !strings.Contains(log, key) || !strings.Contains(log, c.owner) {
				problems = append(problems, "left running: the delete of "+key+" is not said with its owner "+c.owner)
			}
			continue
		}
		if raw == nil {
			problems = append(problems, "left running: the step "+key+" is gone after the sweep, want it kept or ended")
			continue
		}
		s, err := delivery.DecodeStep(raw)
		if err != nil {
			problems = append(problems, key+": "+err.Error())
			continue
		}
		want := c.state
		if want == "" {
			want = delivery.StepRunning
		}
		if c.ended {
			want = delivery.StepFailed
		}
		if s.State != want {
			problems = append(problems, fmt.Sprintf("left running: the step %s (%s, owner %q, %s before the start) is %s after the sweep, want %s", key, c.tool, c.owner, c.age, s.State, want))
		}
		if c.ended && !strings.Contains(log, key) {
			problems = append(problems, "left running: the end of "+key+" is not said")
		}
		if c.ended && c.owner != "" && !strings.Contains(log, c.owner) {
			problems = append(problems, "left running: the end of "+key+" does not name its owner "+c.owner)
		}
	}
	sort.Strings(problems)
	return problems
}

// steps is a steps state with revisions, whose Update lands only at the revision read (stdio.ErrChanged otherwise).
type steps struct {
	mu   sync.Mutex
	vals map[string][]byte
	revs map[string]uint64
}

func (s *steps) set(key string, raw []byte) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.vals[key], s.revs[key] = raw, s.revs[key]+1
}

func (s *steps) get(key string) []byte {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.vals[key]
}

func (s *steps) scan() buildstep.Scan {
	return buildstep.Scan{
		Keys: func() ([]string, error) {
			s.mu.Lock()
			defer s.mu.Unlock()
			var out []string
			for k := range s.vals {
				out = append(out, k)
			}
			return out, nil
		},
		Get: func(key string) (json.RawMessage, uint64, error) {
			s.mu.Lock()
			defer s.mu.Unlock()
			return s.vals[key], s.revs[key], nil
		},
		Delete: func(key string, revision uint64) error {
			s.mu.Lock()
			defer s.mu.Unlock()
			if s.revs[key] != revision {
				return stdio.ErrChanged
			}
			delete(s.vals, key)
			s.revs[key]++
			return nil
		},
		Update: func(key string, value any, revision uint64) (uint64, error) {
			raw, err := json.Marshal(value)
			if err != nil {
				return 0, err
			}
			s.mu.Lock()
			defer s.mu.Unlock()
			if s.revs[key] != revision {
				return 0, stdio.ErrChanged
			}
			s.vals[key], s.revs[key] = raw, revision+1
			return revision + 1, nil
		},
	}
}