mesh-delivery/modules/mesh-delivery-checker/rules/facts.go

// Package rules is the checker's judging rules, the checking program's: given the facts of one check (what the build
// modules, the manifest validator and the composer answered, and what failed in the asking), the verdict (novox/hq ADR
// 0644, ADR 0656; design 47's bootstrap delivery). The running checker asks, waits, keeps and records; it gathers the
// facts and has them judged by the checking program built from the change's heads (cmd/mesh-checking-program), which
// imports this package, so a change to a rule is judged by its own rule.
//
// It is the checker module's own package (novox/hq ADR 0634 §1): no seat carries it and no other module imports it.
package rules

import (
	"encoding/json"
	"fmt"
	"path"
	"sort"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)

// FactsVersion is the version of Facts this program reads. A checking program built from a later change reads the facts
// a running checker of this version writes, so a version is added to, never changed; a program refuses one it does not
// read, naming it (DecodeFacts).
const FactsVersion = 1

// Facts are everything one check judges, in the order the running checker met it.
type Facts struct {
	V int `json:"v"`
	// Group and Heads name the check (meshdeliverychecker.VerdictsByGroupHeadsValue's Group and Heads); Trunk is each repository's trunk head merged.
	Group   string            `json:"group"`
	Heads   string            `json:"heads"`
	Trunk   map[string]string `json:"trunk,omitempty"`
	Members []Member          `json:"members"`
	// SeatDigests are, by seat, the digests (SeatDigest) of the definitions the running checker's reader holds: the
	// `-seats` answer leaves out every head definition that is the same (milestone 47/G, the lead's ruling on review
	// finding 3: a whole list is over a run answer's bound). None from a checker that hands none: then every head
	// definition is answered.
	SeatDigests map[string]string `json:"seat-digests,omitempty"`
	// Series are the changes of the members' modules' series, as structures, put in the verdict whatever it is (DLV-C4).
	Series []meshdeliverychecker.VerdictsByGroupHeadsValueSeriesItem `json:"series,omitempty"`
	// Steps are, in the order met, each level a build module answered and each failure the asking met.
	Steps []Step `json:"steps,omitempty"`
	// CloneMS is how much of the steps' time their levels' running did not take (meshdeliverychecker.VerdictsByGroupHeadsValueTook.Clone).
	CloneMS int64 `json:"clone-ms,omitempty"`
	// Composition is how the check composed, after every step; nil when it was not composed (a narrowed try, a checker
	// that composes nothing): its composition is then not run, unsaid.
	Composition *Composition `json:"composition,omitempty"`
	// Notes are the running checker's notes of the check, for a reader.
	Notes []string `json:"notes,omitempty"`
}

// Member is one pull request of the check.
type Member struct {
	Repository string `json:"repository"`
	Number     int    `json:"number"`
	Head       string `json:"head"`
	Trunk      string `json:"trunk,omitempty"`
}

// Step is one fact of the check, in the order met: the node that ran it, and one of a level its build module answered
// (Level, as the build module gave it), what its fetch read of compiled programs (Programs), the manifest validator's
// answers (Manifests), the module graph's ordering edges (Graph) or a failure the asking met (Failed: a step not asked, refused or not answered, a conflict, worded
// by the running checker). A step with none only says which node ran it.
type Step struct {
	Repository string                 `json:"repository"`
	Number     int                    `json:"number"`
	Module     string                 `json:"module,omitempty"`
	Node       string                 `json:"node,omitempty"`
	Level      *buildstep.LevelAnswer `json:"level,omitempty"`
	Failed     *FailedFirst           `json:"failed,omitempty"`
	Programs   *Programs              `json:"programs,omitempty"`
	Manifests  *Manifests             `json:"manifests,omitempty"`
	// Graph is the module graph's ordering edges, as the change leaves them and as the trunk has them (JudgeGraph).
	Graph *Graph `json:"graph,omitempty"`
	// Renames is what a module brings and drops of `renamed-from` and how the seat mesh-config answered (JudgeRenames).
	Renames *Renames `json:"renames,omitempty"`
	// Unparsed is what the indexer keeps of a pull request's seat definitions that do not parse (JudgeUnparsed).
	Unparsed *Unparsed `json:"unparsed,omitempty"`
}

// LevelFact is a level answer as the facts carry it: only what the rules read (Tally.Level), so a group's facts stay
// within the run input's bound (buildstep.RunInputOf). The digest, the unmet packages and the level's judged program
// are left out, the kept packages are one count, and each test result keeps only the fields Tested names, without its
// packages, bounds and judged program (review round 1 of milestone 47/B0). An answer this cannot read is carried as given,
// for the rules to refuse it.
func LevelFact(la buildstep.LevelAnswer) *buildstep.LevelAnswer {
	out := &buildstep.LevelAnswer{Level: la.Level}
	kept := 0
	for _, k := range la.Kept {
		kept += k.Tests
	}
	if len(la.Kept) > 0 {
		out.Kept = []buildstep.KeptTaken{{Tests: kept}}
	}
	for _, raw := range la.Tests {
		var res Tested
		if err := json.Unmarshal(raw, &res); err != nil {
			out.Tests = append(out.Tests, raw)
			continue
		}
		res.Packages, res.JudgedBy, res.Bound, res.SetupBound, res.OverWhich = nil, "", 0, 0, ""
		small, err := json.Marshal(res)
		if err != nil {
			small = raw
		}
		out.Tests = append(out.Tests, small)
	}
	return out
}

// DecodeFacts reads facts, refusing a version this program does not read, naming it.
func DecodeFacts(data []byte) (Facts, error) {
	var f Facts
	if err := json.Unmarshal(data, &f); err != nil {
		return Facts{}, fmt.Errorf("rules: the facts are not JSON of their shape: %w", err)
	}
	if f.V != FactsVersion {
		return Facts{}, fmt.Errorf("rules: facts of version %d, this checking program reads version %d", f.V, FactsVersion)
	}
	return f, nil
}

// Counts are one level's counts across a check's members.
type Counts struct {
	Run, Kept, Skipped int
	Elapsed, Setup     float64
	Lint               float64
	Over               bool
	Slowest            []TestTime
}

// Tally is a check's counts by level.
type Tally map[string]*Counts

// NewTally is a tally of both levels, at nothing.
func NewTally() Tally {
	return Tally{buildstep.LevelUnit: {}, buildstep.LevelIntegration: {}}
}

// Level judges one level of a member's check step as it was answered: the kept ones taken are counted, and every test
// result is read and counted, also after one that did not pass, so the verdict names every failed test of the level
// (issue 1921). It answers the level's failure (nil when it passed), its running time (its tests' and setup's) and the
// results read, for the running checker to keep the passes from.
func (t Tally) Level(repository string, number int, module, node string, la buildstep.LevelAnswer) (*meshdeliverychecker.VerdictsByGroupHeadsValue, float64, []Tested) {
	a := t[la.Level]
	if a == nil {
		return failAt(node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", repository, number, la.Level)), 0, nil
	}
	for _, k := range la.Kept {
		a.Kept += k.Tests
	}
	elapsed := 0.0
	var results, failed []Tested
	for _, raw := range la.Tests {
		var res Tested
		if err := json.Unmarshal(raw, &res); err != nil {
			return failAt(node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", la.Level, repository, number, err)), elapsed, results
		}
		results = append(results, res)
		elapsed += res.Elapsed + res.Setup + res.Lint
		if module == nodeBuildModule {
			if !res.NoTests {
				a.Run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
			}
		} else {
			a.Run += res.Tests
			a.Skipped += res.Skipped
			a.Slowest = append(a.Slowest, res.Slowest...)
		}
		a.Elapsed += res.Elapsed
		a.Setup += res.Setup
		a.Lint += res.Lint
		a.Over = a.Over || res.OverBound
		if !res.Passed {
			failed = append(failed, res)
		}
	}
	if len(failed) > 0 {
		return LevelFailure(repository, module, la.Level, failed...), elapsed, results
	}
	return nil, elapsed, results
}

// Judge is the verdict of the facts (Judged without its notes).
func Judge(f Facts) meshdeliverychecker.VerdictsByGroupHeadsValue { return Judged(f).Verdict }

// Judged is the answer of the facts: every failure in the order met, joined and settled, or a pass; the levels'
// counts, times, machines and the levels over their bound; then the composition taken in; and the notes the rules give
// a reader (the manifest validator's warnings and acceptances).
func Judged(f Facts) Answer {
	var notes []string
	tally := NewTally()
	var failures []*meshdeliverychecker.VerdictsByGroupHeadsValue
	nodes := map[string]bool{}
	for _, s := range f.Steps {
		if s.Node != "" {
			nodes[s.Node] = true
		}
		switch {
		case s.Failed != nil:
			failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: s.Failed})
		case s.Level != nil:
			if v, _, _ := tally.Level(s.Repository, s.Number, s.Module, s.Node, *s.Level); v != nil {
				failures = append(failures, v)
			}
		case s.Programs != nil:
			if ff := JudgePrograms(s.Node, *s.Programs); ff != nil {
				failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: ff})
			}
		case s.Graph != nil:
			ff, said := JudgeGraph(*s.Graph)
			notes = append(notes, said...)
			if ff != nil {
				failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: ff})
			}
		case s.Renames != nil:
			ffs, said := JudgeRenames(*s.Renames)
			notes = append(notes, said...)
			for _, ff := range ffs {
				failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: &ff})
			}
		case s.Unparsed != nil:
			for _, ff := range JudgeUnparsed(*s.Unparsed) {
				failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: &ff})
			}
		case s.Manifests != nil:
			ff, said := JudgeManifests(*s.Manifests)
			notes = append(notes, said...)
			if ff != nil {
				failures = append(failures, &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: ff})
			}
		}
	}
	v := meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictPass}
	if joined := Settled(JoinFailures(failures...)); joined != nil {
		v = *joined
	}
	v.Group, v.Heads, v.Trunk, v.Series = f.Group, f.Heads, f.Trunk, f.Series
	Outcome(&v, tally, f.CloneMS, nodes)
	if f.Composition != nil {
		JudgeComposition(&v, *f.Composition)
	}
	return Answer{V: AnswerVersion, Verdict: v, Notes: notes}
}

// Outcome fills a verdict's levels (on a pass, or when it names none), times, machines and levels over their bound from
// the tally, and says its composition is not yet run (ADR 0484 clause 6).
func Outcome(v *meshdeliverychecker.VerdictsByGroupHeadsValue, tally Tally, cloneMS int64, nodes map[string]bool) {
	if v.State == meshdeliverycheckerclient.VerdictPass || len(v.Levels) == 0 {
		v.Levels = map[string]meshdeliverychecker.VerdictsByGroupHeadsValueLevelsValue{}
		for level, a := range tally {
			v.Levels[level] = meshdeliverychecker.VerdictsByGroupHeadsValueLevelsValue{Run: int64(a.Run), Kept: int64(a.Kept), Skipped: int64(a.Skipped)}
		}
	}
	ms := func(s float64) int64 { return int64(s * 1000) }
	u, i := tally[buildstep.LevelUnit], tally[buildstep.LevelIntegration]
	v.Took = &meshdeliverychecker.VerdictsByGroupHeadsValueTook{Clone: cloneMS, Unit: ms(u.Elapsed), Lint: ms(u.Lint), IntegrationSetup: ms(i.Setup), IntegrationRun: ms(i.Elapsed)}
	v.Machines = nil
	for node := range nodes {
		v.Machines = append(v.Machines, node)
	}
	sort.Strings(v.Machines)
	v.OverBound = nil
	for _, level := range []string{buildstep.LevelUnit, buildstep.LevelIntegration} {
		if tally[level].Over {
			v.OverBound = append(v.OverBound, level)
		}
	}
	v.Composition = &meshdeliverychecker.VerdictsByGroupHeadsValueComposition{State: meshdeliverycheckerclient.CompositionNotRun}
}

// Slowest are, of each level over its bound, its three slowest tests, worded for the status line.
func Slowest(tally Tally) []string {
	var slow []string
	for _, level := range []string{buildstep.LevelUnit, buildstep.LevelIntegration} {
		a := tally[level]
		if !a.Over {
			continue
		}
		sort.SliceStable(a.Slowest, func(x, y int) bool { return a.Slowest[x].Elapsed > a.Slowest[y].Elapsed })
		for n, s := range a.Slowest {
			if n == 3 {
				break
			}
			slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
		}
	}
	return slow
}

func failAt(machine, words string) *meshdeliverychecker.VerdictsByGroupHeadsValue {
	return &meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: &FailedFirst{Machine: machine, Words: words}}
}

// AnswerVersion is the version of Answer this program writes.
const AnswerVersion = 1

// Answer is what the checking program answers on the last line of its standard output: its version, the verdict and the
// notes its rules give a reader of the check.
type Answer struct {
	V       int                                           `json:"v"`
	Verdict meshdeliverychecker.VerdictsByGroupHeadsValue `json:"verdict"`
	Notes   []string                                      `json:"notes,omitempty"`
}