package conformance import ( "context" "encoding/json" "fmt" "sort" "strings" "sync" "testing" "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" ) // **A step left running by a previous process** (novox/hq issue 1890, note 21): when a delivery replaced mesh-build-go // on a node while a check ask ran, the new process's start sweep let the ask's key stand as running, and it held one // of the node's build slots until a hand end. Every build module runs this case (LeftRunning) through the start sweep // its main runs: the hand-started steps a previous process left are ended at once and loud, its asks' keys deleted (the // stream redelivers the ask, novox/hq ADR 0525 §6, and its asker keeps waiting for that), what this process runs kept. A // catalogue check (mesh-catalog checks/failurekit) refuses a build module whose tests do not call LeftRunning. // Sweep is a build module's own start sweep, as its main runs it once at its start: it ends node's steps in scan that // no process of the module runs any more (buildstep.EndStaleAtStart, here with the runtime already serving), before // being the start of the process, saying each end with logf. type Sweep func(ctx context.Context, scan buildstep.Scan, node string, before time.Time, logf func(string, ...any)) // LeftNode is the node the case's steps are of. const LeftNode = "kitnode" // leftCase is one step in the steps state before the sweep, and whether the sweep must end it. type leftCase struct { id, tool, owner, state string node string age time.Duration // started this long before the process began (negative: after it) ended bool deleted bool // a gone process's ask: its key is deleted, not ended } // gone is a previous process's owner on the same node. var gone = "b00t-of-the-kit.4242.mvp0000" func leftCases() []leftCase { return []leftCase{ {id: "mvpwlkzi7e6ca8", tool: "check ask", owner: gone, age: 5 * time.Minute, deleted: true}, // note 21's ghost {id: "mvpwlkzi7e6ca9", tool: "delivery ask", owner: gone, age: -time.Second, deleted: true}, {id: "gonehand1", tool: "build_x_test", owner: gone, age: time.Minute, ended: true}, {id: "oldhand1", tool: "build_x_fetch", age: time.Hour, ended: true}, // no owner: an older build's {id: "mvpwlkzi7e6caa", tool: "check ask", age: time.Hour, ended: true}, // no owner, past the bound {id: "mvpwlkzi7e6cab", tool: "check ask", age: 5 * time.Minute}, // no owner, the stream's still {id: "mvpwlkzi7e6cac", tool: "check ask", owner: buildstep.Owner, age: time.Minute}, // this process's {id: "minehand1", tool: "build_x_test", owner: buildstep.Owner, age: -time.Second}, {id: "donehand1", tool: "build_x_test", owner: gone, age: time.Hour, state: delivery.StepPassed}, {id: "otherhand1", tool: "build_x_test", owner: gone, age: time.Hour, node: "kitother"}, } } // LeftRunning runs the case through sweep and fails the test for each step the sweep ended or kept wrongly, or ended // without saying it. func LeftRunning(t *testing.T, sweep Sweep) { t.Helper() for _, problem := range JudgeLeftRunning(sweep) { t.Error(problem) } } // JudgeLeftRunning says what is wrong with what sweep did to the case; nothing when it keeps the rule. func JudgeLeftRunning(sweep Sweep) []string { st := &steps{vals: map[string][]byte{}, revs: map[string]uint64{}} before := time.Now() for _, c := range leftCases() { node, state := c.node, c.state if node == "" { node = LeftNode } if state == "" { state = delivery.StepRunning } key, _ := delivery.StepKey(node, c.id) raw, err := delivery.EncodeStep(delivery.Step{ID: c.id, Tool: c.tool, Node: node, State: state, Started: before.Add(-c.age).UTC(), Owner: c.owner}) if err != nil { return []string{"the kit's step " + c.id + ": " + err.Error()} } st.set(key, raw) } served := make(chan struct{}) close(served) was := buildstep.StaleServed buildstep.StaleServed = func() <-chan struct{} { return served } defer func() { buildstep.StaleServed = was }() var mu sync.Mutex var said []string ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() sweep(ctx, st.scan(), LeftNode, before, func(f string, a ...any) { mu.Lock() defer mu.Unlock() said = append(said, fmt.Sprintf(f, a...)) }) mu.Lock() log := strings.Join(said, "\n") mu.Unlock() var problems []string for _, c := range leftCases() { node := c.node if node == "" { node = LeftNode } key, _ := delivery.StepKey(node, c.id) raw := st.get(key) if c.deleted { if raw != nil { problems = append(problems, fmt.Sprintf("left running: the ask %s of the gone owner %q is still held after the sweep (%s); "+ "its key is deleted, never ended: the stream redelivers it (ADR 0525 §6)", key, c.owner, raw)) } if !strings.Contains(log, key) || !strings.Contains(log, c.owner) { problems = append(problems, "left running: the delete of "+key+" is not said with its owner "+c.owner) } continue } if raw == nil { problems = append(problems, "left running: the step "+key+" is gone after the sweep, want it kept or ended") continue } s, err := delivery.DecodeStep(raw) if err != nil { problems = append(problems, key+": "+err.Error()) continue } want := c.state if want == "" { want = delivery.StepRunning } if c.ended { want = delivery.StepFailed } if s.State != want { problems = append(problems, fmt.Sprintf("left running: the step %s (%s, owner %q, %s before the start) is %s after the sweep, want %s", key, c.tool, c.owner, c.age, s.State, want)) } if c.ended && !strings.Contains(log, key) { problems = append(problems, "left running: the end of "+key+" is not said") } if c.ended && c.owner != "" && !strings.Contains(log, c.owner) { problems = append(problems, "left running: the end of "+key+" does not name its owner "+c.owner) } } sort.Strings(problems) return problems } // steps is a steps state with revisions, whose Update lands only at the revision read (stdio.ErrChanged otherwise). type steps struct { mu sync.Mutex vals map[string][]byte revs map[string]uint64 } func (s *steps) set(key string, raw []byte) { s.mu.Lock() defer s.mu.Unlock() s.vals[key], s.revs[key] = raw, s.revs[key]+1 } func (s *steps) get(key string) []byte { s.mu.Lock() defer s.mu.Unlock() return s.vals[key] } func (s *steps) scan() buildstep.Scan { return buildstep.Scan{ Keys: func() ([]string, error) { s.mu.Lock() defer s.mu.Unlock() var out []string for k := range s.vals { out = append(out, k) } return out, nil }, Get: func(key string) (json.RawMessage, uint64, error) { s.mu.Lock() defer s.mu.Unlock() return s.vals[key], s.revs[key], nil }, Delete: func(key string, revision uint64) error { s.mu.Lock() defer s.mu.Unlock() if s.revs[key] != revision { return stdio.ErrChanged } delete(s.vals, key) s.revs[key]++ return nil }, Update: func(key string, value any, revision uint64) (uint64, error) { raw, err := json.Marshal(value) if err != nil { return 0, err } s.mu.Lock() defer s.mu.Unlock() if s.revs[key] != revision { return 0, stdio.ErrChanged } s.vals[key], s.revs[key] = raw, revision+1 return revision + 1, nil }, } }