Merge pull request 'The delivery view shows pipelines live (issue 1983, H-A-467)' (#327) from feat/pipelines-view into main

This commit was merged in pull request #327.
This commit is contained in:
2026-11-22 13:41:03 +00:00
19 changed files with 1478 additions and 49 deletions
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 6
const Form = 7
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "6"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=64600be410e9 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
generatedForms = "7"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
)
const (
@@ -73,7 +73,7 @@ func VerbDescription(verb string) string {
case VerbShow:
return "One pipeline: naming a mover, its steps alone; with no mover, each mover's row (its module, outcome and failed step), never every step of every mover, or the steps of any mover the step and state filters hold; and its step log, with the stores' fill (design 47 H2 §789, H-A-376). Reads only."
case VerbList:
return "The pipelines and runs of a module, a branch or a commit, newest first: by default one short row per pipeline, the newest 20 (its key, call, start, end, outcome, movers' count and first failed or could-not-run step with its module and why); the whole pipelines, each record with its movers and steps counted, never its steps (show gives those), only with full and a limit (design 47 H2 H-A-376). Reads only."
return "The pipelines and runs of a module, a branch or a commit, newest first: by default one short row per pipeline, the newest 20 (its key, call, source, revision, start, end, outcome, movers' count, its steps counted per outcome and first failed or could-not-run step with its module and why); the whole pipelines, each record with its movers and steps counted, never its steps (show gives those), only with full and a limit (design 47 H2 H-A-376, H-A-467). Reads only."
case VerbStands:
return "What stands still, and since when: each step not ended and each wait held. Reads only."
case VerbWhyCarried:
@@ -879,7 +879,7 @@ const (
PipelineEndedOutcomeCancelled = "cancelled"
)
// PipelineRow is one pipeline in list's short form (design 47 H2 H-A-376).
// PipelineRow is one pipeline in list's short form, with its source, revision and its steps counted per outcome (design 47 H2 H-A-376, H-A-467).
type PipelineRow struct {
// the merge call whose source revision the take-in took in
Call string `json:"call,omitempty"`
@@ -889,8 +889,14 @@ type PipelineRow struct {
MoverCount int64 `json:"mover-count"`
Outcome string `json:"outcome"`
// the pipeline's key
Pipeline string `json:"pipeline"`
Started *time.Time `json:"started,omitempty"`
Pipeline string `json:"pipeline"`
// the catalog graph revision of the branch the pipeline runs for (design 47 H2 H-A-467)
Revision int64 `json:"revision,omitempty"`
// the source revision of the watched branch the pipeline runs for, its key (design 47 H2 H-A-467)
Source int64 `json:"source,omitempty"`
Started *time.Time `json:"started,omitempty"`
// how many of the pipeline's steps hold each outcome, by the step's outcome (design 47 H2 H-A-467)
Steps map[string]int64 `json:"steps,omitempty"`
}
// The values of PipelineRow's outcome.
@@ -171,7 +171,7 @@
},
{
"name": "list",
"description": "The pipelines and runs of a module, a branch or a commit, newest first: by default one short row per pipeline, the newest 20 (its key, call, start, end, outcome, movers' count and first failed or could-not-run step with its module and why); the whole pipelines, each record with its movers and steps counted, never its steps (show gives those), only with full and a limit (design 47 H2 H-A-376). Reads only.",
"description": "The pipelines and runs of a module, a branch or a commit, newest first: by default one short row per pipeline, the newest 20 (its key, call, source, revision, start, end, outcome, movers' count, its steps counted per outcome and first failed or could-not-run step with its module and why); the whole pipelines, each record with its movers and steps counted, never its steps (show gives those), only with full and a limit (design 47 H2 H-A-376, H-A-467). Reads only.",
"input": {
"type": "object",
"properties": {
@@ -2275,6 +2275,14 @@
"type": "string",
"description": "the merge call whose source revision the take-in took in"
},
"source": {
"type": "integer",
"description": "the source revision of the watched branch the pipeline runs for, its key (design 47 H2 H-A-467)"
},
"revision": {
"type": "integer",
"description": "the catalog graph revision of the branch the pipeline runs for (design 47 H2 H-A-467)"
},
"started": {
"type": "string",
"format": "date-time"
@@ -2301,9 +2309,16 @@
},
"failed": {
"$ref": "#/defs/FailedStep"
},
"steps": {
"type": "object",
"additionalProperties": {
"type": "integer"
},
"description": "how many of the pipeline's steps hold each outcome, by the step's outcome (design 47 H2 H-A-467)"
}
},
"description": "one pipeline in list's short form (design 47 H2 H-A-376)",
"description": "one pipeline in list's short form, with its source, revision and its steps counted per outcome (design 47 H2 H-A-376, H-A-467)",
"required": [
"pipeline",
"outcome",
@@ -2338,5 +2353,5 @@
]
}
},
"form": 6
"form": 7
}
@@ -420,7 +420,7 @@ export interface PipelineEnded {
"why"?: string;
}
/** one pipeline in list's short form (design 47 H2 H-A-376) */
/** one pipeline in list's short form, with its source, revision and its steps counted per outcome (design 47 H2 H-A-376, H-A-467) */
export interface PipelineRow {
/** the merge call whose source revision the take-in took in */
"call"?: string;
@@ -431,7 +431,13 @@ export interface PipelineRow {
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
/** the pipeline's key */
"pipeline": string;
/** the catalog graph revision of the branch the pipeline runs for (design 47 H2 H-A-467) */
"revision"?: number;
/** the source revision of the watched branch the pipeline runs for, its key (design 47 H2 H-A-467) */
"source"?: number;
"started"?: string;
/** how many of the pipeline's steps hold each outcome, by the step's outcome (design 47 H2 H-A-467) */
"steps"?: Record<string, number>;
}
/** a pipeline's start, said at its write, a waiting one's too (design 47 H §746, H-A-52) */
@@ -286,12 +286,27 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
}
rows := make([]seat.PipelineRow, 0, len(out))
for _, p := range out {
rows = append(rows, seat.PipelineRow{Pipeline: p.Pipeline, Call: p.Call, Started: p.Started, Ended: p.Ended, Outcome: p.Outcome,
MoverCount: p.MoverCount, Failed: firstFailed(steps[p.Pipeline])})
// A row carries its source, its revision and its steps counted per outcome (design 47 H2 H-A-467): a page's list is
// one read of the short form.
rows = append(rows, seat.PipelineRow{Pipeline: p.Pipeline, Call: p.Call, Source: p.Source, Revision: p.Revision,
Started: p.Started, Ended: p.Ended, Outcome: p.Outcome, MoverCount: p.MoverCount, Steps: stepCounts(steps[p.Pipeline]),
Failed: firstFailed(steps[p.Pipeline])})
}
return seat.ListAnswer{Rows: rows}, nil
}
// stepCounts is how many of steps hold each outcome (design 47 H2 H-A-467), nil for none.
func stepCounts(steps []seat.StepResult) map[string]int64 {
if len(steps) == 0 {
return nil
}
out := map[string]int64{}
for _, s := range steps {
out[s.Outcome]++
}
return out
}
// Stands is every step that has started and not ended, with what it waits on and since when (§75, §83): the watchman
// seeds its open waits from it.
func (r Reads) Stands() (seat.StandsAnswer, error) {
@@ -67,7 +67,7 @@ func TestReads(t *testing.T) {
Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}},
Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}})
// A pipeline that failed: list's row names its first failed step (design 47 H2 H-A-376).
put(t, pipes, "gone.2", seat.Pipeline{Pipeline: "gone.2", Branch: "gone", Call: "call-2", Outcome: seat.PipelineOutcomeFailed, Started: &t0, Ended: &t1,
put(t, pipes, "gone.2", seat.Pipeline{Pipeline: "gone.2", Branch: "gone", Call: "call-2", Source: 2, Revision: 5, Outcome: seat.PipelineOutcomeFailed, Started: &t0, Ended: &t1,
Movers: []seat.Mover{{Module: "zeta", Commit: "g2"}}, Steps: []seat.StepResult{
{Module: "zeta", Step: seat.StepResultStepLint, Artifact: "zeta:program", Status: "optional", Outcome: seat.StepResultOutcomeFailed},
{Module: "zeta", Step: seat.StepResultStepUnit, Artifact: "zeta:program", Status: "required", Outcome: seat.StepResultOutcomeCouldNotRun, Why: "its node lacks node-build-node"}}})
@@ -150,13 +150,19 @@ func TestReads(t *testing.T) {
a, err := r.List(seat.ListInput{Branch: "main", Limit: 1})
return names(a.Rows), err
}, "main.7.gamma"},
// Design 47 H2 H-A-376: a row is the pipeline's key, call, start, end, outcome, movers' count and first failed step;
// Design 47 H2 H-A-376, H-A-467: a row is the pipeline's key, call, source, revision, start, end, outcome, movers' count,
// its steps per outcome and first failed step;
// the whole pipelines are answered only with full and a limit; with no limit the rows are the newest 20.
{"list's row of a failed pipeline", func() (string, error) {
a, err := r.List(seat.ListInput{Branch: "gone"})
raw, _ := json.Marshal(a.Rows)
return fmt.Sprint(string(raw), " ", len(a.Pipelines)), err
}, `[{"call":"call-2","ended":"2026-11-11T10:01:00Z","failed":{"artifact":"zeta:program","module":"zeta","outcome":"could not run","step":"unit","why":"its node lacks node-build-node"},"mover-count":1,"outcome":"failed","pipeline":"gone.2","started":"2026-11-11T10:00:00Z"}] 0`},
}, `[{"call":"call-2","ended":"2026-11-11T10:01:00Z","failed":{"artifact":"zeta:program","module":"zeta","outcome":"could not run","step":"unit","why":"its node lacks node-build-node"},"mover-count":1,"outcome":"failed","pipeline":"gone.2","revision":5,"source":2,"started":"2026-11-11T10:00:00Z","steps":{"could not run":1,"failed":1}}] 0`},
// Design 47 H2 H-A-467: a row counts the steps per outcome, a mover's kept steps (H-A-187) with them.
{"list's row counts its steps per outcome", func() (string, error) {
a, err := r.List(seat.ListInput{Branch: "next"})
return fmt.Sprint(a.Rows[0].Source, " ", a.Rows[0].Steps), err
}, "8 map[passed:2 waiting:1]"},
{"list's full needs a limit", func() (string, error) {
_, err := r.List(seat.ListInput{Full: true})
return "", err
@@ -3,6 +3,8 @@
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve). `holds`
// lists every fingerprint the assigner holds with its node, cause and answer; `hold-release <module> <fingerprint> [words]` and
// `hold-keep <module> <fingerprint> [words]` answer its hold ask, again when the operator changes their mind (issue 1906).
// `pipelines [branch]` and `pipeline <pipeline> [mover]` are the pipelines page (design 47 H2 H-A-467): a branch's newest
// pipelines, one pipeline's mover rows or one mover's steps; with -follow printed again at each change the runner's events say.
package main
import (
@@ -13,7 +15,9 @@ import (
"os"
"strings"
"text/tabwriter"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"mesh-delivery-view/internal/view"
)
@@ -31,9 +35,19 @@ func run(args []string, socket string, out, errOut io.Writer) int {
fs := flag.NewFlagSet("mesh-delivery-cli", flag.ContinueOnError)
fs.SetOutput(errOut)
asJSON := fs.Bool("json", false, "print the rows as JSON")
follow := fs.Bool("follow", false, "pipelines and pipeline: print again at each change the runner's events say, until interrupted")
next := fs.String("next", "", "pipeline: the cursor of the next page a pipeline's answer named")
limit := fs.Int("limit", view.DefaultPipelinesLimit, "pipelines: the newest this many pipelines of the branch")
if err := fs.Parse(args); err != nil {
return 2
}
if fs.NArg() > 0 && (fs.Arg(0) == "pipelines" || fs.Arg(0) == "pipeline") {
if *limit < 1 {
fmt.Fprintln(errOut, "mesh-delivery-cli: -limit is a number of at least one")
return 2
}
return pipelines(socket, fs.Args(), *next, *limit, *follow, *asJSON, out, errOut)
}
if fs.NArg() > 0 && fs.Arg(0) == "holds" {
return holds(socket, *asJSON, out, errOut)
}
@@ -53,7 +67,7 @@ func run(args []string, socket string, out, errOut io.Writer) int {
}
if fs.NArg() > 0 {
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]; holds; hold-release|hold-keep <module> <fingerprint> [words]")
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]; holds; hold-release|hold-keep <module> <fingerprint> [words]; [-follow] [-limit <n>] pipelines [branch]; [-follow] [-next <cursor>] pipeline <pipeline> [mover]")
return 2
}
key, option := fs.Arg(1), fs.Arg(2)
@@ -82,6 +96,61 @@ func run(args []string, socket string, out, errOut io.Writer) int {
return 0
}
// pipelines prints the pipelines page (design 47 H2 H-A-467): `pipelines [branch]` the newest pipelines of a branch (main by
// default), `pipeline <pipeline> [mover]` one pipeline's mover rows or one mover's steps; with -follow again at each change.
func pipelines(socket string, args []string, next string, limit int, follow, asJSON bool, out, errOut io.Writer) int {
var branch, pipeline, mover string
switch {
case args[0] == "pipelines" && len(args) <= 2:
if len(args) == 2 {
branch = args[1]
}
case args[0] == "pipeline" && (len(args) == 2 || len(args) == 3):
pipeline = args[1]
if len(args) == 3 {
mover = args[2]
}
default:
fmt.Fprintln(errOut, "mesh-delivery-cli [-follow] [-limit <n>] pipelines [branch]; mesh-delivery-cli [-follow] [-next <cursor>] pipeline <pipeline> [mover]")
return 2
}
var failed error
err := view.AskPipelines(socket, branch, limit, pipeline, mover, next, follow, func(page view.PipelinesPage, show *seatShow) bool {
if follow {
fmt.Fprintf(out, "--- %s ---\n", time.Now().Format("15:04:05"))
}
if asJSON {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
var v any = page
if show != nil {
v = show
}
if failed = enc.Encode(v); failed != nil {
return false
}
return true
}
if show != nil {
view.PipelineTable(out, *show, mover)
} else {
view.PipelinesTable(out, page)
}
return true
})
if err == nil {
err = failed
}
if err != nil {
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
return 1
}
return 0
}
// seatShow is the seat's answer of show, as the view hands it on.
type seatShow = seat.ShowAnswer
// holds prints every hold: module, fingerprint, node, since, answer, words.
func holds(socket string, asJSON bool, out, errOut io.Writer) int {
rows, err := view.AskHolds(socket)
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
pseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
)
@@ -304,3 +305,63 @@ func rawAsk(t *testing.T, a oc.Ask) json.RawMessage {
}
const testView = "acme-view"
// Design 47 H2 H-A-467: the terminal view's pipelines page: a branch's newest pipelines, and one pipeline's mover rows.
func TestThePipelinesCommands(t *testing.T) {
m := view.New(testView)
call := func(key string, body any) (json.RawMessage, error) {
switch key {
case view.PipelinesListVerb:
in := body.(pseat.ListInput)
if in.Branch != "next" {
return json.Marshal(pseat.ListAnswer{})
}
rows := []pseat.PipelineRow{{Pipeline: "next.3", Source: 2, Revision: 3, Outcome: "passed", MoverCount: 1, Steps: map[string]int64{"passed": 4}},
{Pipeline: "next.2", Source: 2, Revision: 2, Outcome: "superseded", MoverCount: 1}}
return json.Marshal(pseat.ListAnswer{Rows: rows[:min(int(in.Limit), len(rows))]})
case view.PipelinesShowVerb:
return json.Marshal(pseat.ShowAnswer{Pipeline: pseat.Pipeline{Pipeline: "next.3", Outcome: "passed", Movers: []pseat.Mover{{Module: "alpha", Fingerprint: "src2:aa", Version: "0.2.1"}}},
Movers: []pseat.MoverRow{{Module: "alpha", Outcome: "passed"}}})
}
return nil, nil
}
m.SetPipelines(view.NewPipelines(call, nil, 0, nil))
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "p")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
l, err := view.ListenSocket(filepath.Join(dir, "cli.sock"))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
go view.ServeSocket(ctx, l, m)
path := filepath.Join(dir, "cli.sock")
for _, r := range []struct {
args []string
want string
}{
{[]string{"pipelines", "next"}, "next.3 3 2 passed 1 4 0 0 0 0 0 0 0"},
{[]string{"pipelines", "next"}, "next.2"}, // the newest 20 by default
{[]string{"-limit", "1", "pipelines", "next"}, "next.3"},
{[]string{"pipeline", "next.3"}, "alpha src2:aa 0.2.1 passed"},
{[]string{"-json", "pipeline", "next.3"}, `"fingerprint": "src2:aa"`},
} {
var out, errOut bytes.Buffer
if code := run(r.args, path, &out, &errOut); code != 0 || !strings.Contains(strings.Join(strings.Fields(out.String()), " "), strings.Join(strings.Fields(r.want), " ")) {
t.Errorf("%v = %d %q %q, want it to hold %q", r.args, code, out.String(), errOut.String(), r.want)
}
}
var out, errOut bytes.Buffer
if code := run([]string{"-limit", "1", "pipelines", "next"}, path, &out, &errOut); code != 0 || strings.Contains(out.String(), "next.2") {
t.Errorf("-limit 1 = %d %q, want only the newest", code, out.String())
}
for _, args := range [][]string{{"pipeline"}, {"-limit", "0", "pipelines"}} {
out.Reset()
if code := run(args, path, &out, &errOut); code != 2 {
t.Errorf("%v = %d, want 2", args, code)
}
}
}
@@ -27,9 +27,11 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
pclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
pseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -888,6 +890,78 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
if stored, gerr := kvs[viewstate.BucketsOf(testView).Reach].Get(ctx, re.Key); gerr != nil || string(stored.Value()) != string(re.Reach) {
t.Errorf("the event carries %s, the state reach holds %v (%v)", re.Reach, stored, gerr)
}
// Design 47 H2 H-A-467: the pipelines page reads the seat mesh-pipeline-runner's list, again at the runner's event heard on
// the module's one pull, and the delivered source revision links to the newest pipeline of its source.
pipelines := pipelinesRunner(t, nc)
pipelines.set(pseat.PipelineRow{Pipeline: "release.9", Source: rev, Revision: 9, Outcome: "passed", MoverCount: 1},
pseat.PipelineRow{Pipeline: "release.8", Source: rev, Revision: 8, Outcome: "superseded", MoverCount: 1})
var seen [][]string
err = view.AskPipelines(env["MESH_DELIVERY_SOCKET"], catalogBranch, 0, "", "", "", true, func(page view.PipelinesPage, _ *pseat.ShowAnswer) bool {
var keys []string
for _, r := range page.Rows {
keys = append(keys, r.Pipeline)
}
seen = append(seen, keys)
if len(seen) == 1 {
pipelines.set(pseat.PipelineRow{Pipeline: "release.10", Source: rev + 1, Revision: 10, Outcome: "running", MoverCount: 1},
pseat.PipelineRow{Pipeline: "release.9", Source: rev, Revision: 9, Outcome: "passed", MoverCount: 1})
body, _ := json.Marshal(pseat.PipelineEndedEvent{Pipeline: "release.9", Branch: catalogBranch, Source: rev, Revision: 9, Outcome: "passed"})
if _, err := js.Publish(ctx, pclient.RevisionSubject(pseat.EventPipelineEnded, catalogBranch, 9), body); err != nil {
t.Error(err)
return false
}
}
return len(seen) < 2 || !slices.Contains(keys, "release.10")
})
if err != nil || len(seen) < 2 || seen[len(seen)-1][0] != "release.10" {
t.Fatalf("the pipelines followed = %v, %v; want release.10 after the runner's event", seen, err)
}
snap, err := view.Ask(env["MESH_DELIVERY_SOCKET"])
if err != nil {
t.Fatal(err)
}
for _, src := range snap.Sources {
if src.Branch == catalogBranch && src.Pipeline != "release.9" {
t.Errorf("the delivery of %s's source revision %d links to %q, want release.9", src.Branch, src.Revision, src.Pipeline)
}
}
}
// fakeList is the seat mesh-pipeline-runner's verb list as its holder answers it on the bus, from the rows the test sets.
type fakeList struct {
mu sync.Mutex
rows []pseat.PipelineRow
}
func (f *fakeList) set(rows ...pseat.PipelineRow) {
f.mu.Lock()
defer f.mu.Unlock()
f.rows = rows
}
func pipelinesRunner(t *testing.T, nc *nats.Conn) *fakeList {
t.Helper()
f := &fakeList{}
subject := "mesh.seat." + pseat.Seat + ".call." + pseat.VerbList
for _, s := range []string{subject, subject + ".>"} {
sub, err := nc.Subscribe(s, func(m *nats.Msg) {
var in pseat.ListInput
_ = json.Unmarshal(m.Data, &in)
f.mu.Lock()
a := pseat.ListAnswer{}
if in.Branch == catalogBranch && in.Limit == view.DefaultPipelinesLimit {
a.Rows = f.rows
}
f.mu.Unlock()
raw, _ := json.Marshal(map[string]any{"result": a, "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
return f
}
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
@@ -15,6 +15,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"io"
"io/fs"
"log"
"maps"
"mesh-delivery-view/internal/view"
@@ -22,6 +23,7 @@ import (
"os"
"os/signal"
"slices"
"strconv"
"strings"
"sync"
"syscall"
@@ -191,6 +193,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearWaits(ctx, hub, js, own.Waits, logger.Printf); err != nil {
return fmt.Errorf("the waits' events are not handled: %w", err)
}
// The pipelines page reads the seat mesh-pipeline-runner's list and show, and reads again on the runner's events heard on
// the module's one pull, never on a timer (design 47 H2 H-A-467); the watched branches' lists link each delivery.
heard, err := numberFile(getenv("MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"), view.DefaultPipelinesHeard)
if err != nil {
return fmt.Errorf("the setting pipelines-heard: %w", err)
}
pipes := view.NewPipelines(calls.Call, branches.Followed, heard, logger.Printf)
model.SetPipelines(pipes)
if err := hearPipelines(hub, pipes, logger.Printf); err != nil {
return fmt.Errorf("the runner's events are not handled: %w", err)
}
go pipes.Run(ctx)
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
logger.Printf("the consumer is not heard yet (tried again until it is): %v", err)
}
@@ -536,3 +550,27 @@ func (c *catalogModules) apply(module string, entry meshcatalog.CatalogEntry, li
e.History = true
c.model.Apply(e)
}
// numberFile reads a setting the node-engine writes into a file of the state directory: def when path is empty, the file
// missing or empty; an error when it holds no number of at least one.
func numberFile(path string, def int) (int, error) {
if path == "" {
return def, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return def, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return def, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a number of at least one", path, v)
}
return n, nil
}
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"strings"
"testing"
@@ -17,6 +19,37 @@ func TestRunNeedsItsCredential(t *testing.T) {
}
}
// The setting pipelines-heard is read from the file the node-engine writes, its default where none is written (principle 11).
func TestNumberFileReadsASetting(t *testing.T) {
dir := t.TempDir()
write := func(name, v string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(v), 0o600); err != nil {
t.Fatal(err)
}
return p
}
for _, r := range []struct {
name, path string
want int
fails bool
}{
{"no file named is the default", "", 1024, false},
{"a file not written is the default", filepath.Join(dir, "none"), 1024, false},
{"an empty file is the default", write("empty", "\n"), 1024, false},
{"a number is read", write("seven", "7\n"), 7, false},
{"zero is refused", write("zero", "0"), 0, true},
{"words are refused", write("words", "many"), 0, true},
} {
t.Run(r.name, func(t *testing.T) {
n, err := numberFile(r.path, 1024)
if (err != nil) != r.fails || n != r.want {
t.Errorf("numberFile = %d, %v; want %d, failing %v", n, err, r.want, r.fails)
}
})
}
}
// rawAsk is the router's ask as the JSON a gate's or an assigner's value carries (mesh-sdk keeps it raw, ADR 0627 section 4).
func rawAsk(t *testing.T, a oc.Ask) json.RawMessage {
t.Helper()
@@ -7,6 +7,8 @@ import (
"fmt"
"strings"
"mesh-delivery-view/internal/view"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
@@ -92,3 +94,20 @@ func hearWaits(ctx context.Context, hub *process.EventHub, js jetstream.JetStrea
}
return nil
}
// hearPipelines registers the handlers of the runner's events the pipelines page reads again on (design 47 H2 H-A-467): each
// is acknowledged as heard, since the page reads the runner's verbs and keeps nothing of the event; one that names no
// pipeline is said and acknowledged, as no delivery of it again could name one.
func hearPipelines(hub *process.EventHub, pipes *view.Pipelines, logf func(string, ...any)) error {
for _, pattern := range view.PipelineEventPatterns() {
if err := hub.Handle(pattern, func(m jetstream.Msg) {
if err := pipes.Hear(m.Data()); err != nil {
logf("the runner's event %s is not read: %v", m.Subject(), err)
}
_ = m.Ack()
}); err != nil {
return err
}
}
return nil
}
+2
View File
@@ -13,9 +13,11 @@ require (
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
@@ -0,0 +1,293 @@
package view
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"sync"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// The seat mesh-pipeline-runner's verbs list and show as the view's invokes names them: the pipelines page reads the
// pipelines through them and keeps no state of its own (design 47 H2 H-A-467 (1)).
const (
PipelinesListVerb = "seat:" + seat.Seat + "." + seat.VerbList
PipelinesShowVerb = "seat:" + seat.Seat + "." + seat.VerbShow
)
// The pipelines page's defaults (design 47 H2 H-A-467 (1)): the branch main and the newest 20.
const (
DefaultPipelinesBranch = "main"
DefaultPipelinesLimit = 20
// DefaultPipelinesHeard is the default of the setting pipelines-heard: how many pipelines' last changes the view keeps
// for its pages before it forgets them all (principle 11).
DefaultPipelinesHeard = 1024
)
// PipelineEvents are the runner's events the pipelines page refreshes on (design 47 H2 H-A-467 (1)), never a timer.
var PipelineEvents = []string{seat.EventPipelineStarted, seat.EventStepEnded, seat.EventPipelineEnded, seat.EventRePinned}
// PipelineEventPatterns are the subjects of PipelineEvents on the process's hub: each event, with and without the names
// its sayer adds after the event's name.
func PipelineEventPatterns() []string {
var out []string
for _, e := range PipelineEvents {
out = append(out, "mesh.seat."+seat.Seat+".event."+e, "mesh.seat."+seat.Seat+".event."+e+".>")
}
return out
}
// PipelinesPage is one branch's list as the page shows it: the rows list answered, or why it could not be read.
type PipelinesPage struct {
Branch string `json:"branch"`
Rows []seat.PipelineRow `json:"rows"`
Error string `json:"error,omitempty"`
}
// Pipelines reads the runner's pipelines through its seat's verbs and says when they changed, on the events it hears
// (design 47 H2 H-A-467). It holds the last list it read of each branch it follows, the answer of a bus call and no state
// of the view's own: what it read is read again at each event.
type Pipelines struct {
call func(key string, body any) (json.RawMessage, error)
branches func() []string // the watched branches followed beside the default; nil for none
logf func(string, ...any)
dirty chan struct{}
mu sync.Mutex
lists map[string]PipelinesPage // by branch, the branches followed
subs map[chan string]struct{}
version uint64
heard map[string]uint64 // the version of the last change of each pipeline heard
most int // the pipelines heard kept before heard is forgotten whole: the setting pipelines-heard
}
// NewPipelines is a reader of the pipelines through call; branches, when given, names the watched branches whose list is
// followed beside the default branch's, for the deliveries' links (design 47 H2 H-A-467 (3)); heard is the setting
// pipelines-heard, its default below one.
func NewPipelines(call func(key string, body any) (json.RawMessage, error), branches func() []string, heard int, logf func(string, ...any)) *Pipelines {
if logf == nil {
logf = func(string, ...any) {}
}
if heard < 1 {
heard = DefaultPipelinesHeard
}
return &Pipelines{call: call, branches: branches, logf: logf, dirty: make(chan struct{}, 1),
lists: map[string]PipelinesPage{}, subs: map[chan string]struct{}{}, heard: map[string]uint64{}, most: heard}
}
// List asks the seat's verb list for one branch's newest limit pipelines in its short form; a limit below one is the
// default's.
func (p *Pipelines) List(branch string, limit int) ([]seat.PipelineRow, error) {
if limit < 1 {
limit = DefaultPipelinesLimit
}
raw, err := p.call(PipelinesListVerb, seat.ListInput{Branch: branch, Limit: int64(limit)})
if err != nil {
return nil, fmt.Errorf("%s of %s: %w", PipelinesListVerb, branch, err)
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("%s of %s answered: %s", PipelinesListVerb, branch, said)
}
var a seat.ListAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return nil, fmt.Errorf("the answer of %s of %s is not a list: %w", PipelinesListVerb, branch, err)
}
return a.Rows, nil
}
// Show asks the seat's verb show for one pipeline: naming no mover, its movers' rows; naming one, that mover's steps
// (design 47 H2 H-A-467 (2)); next is the cursor of a page show answered.
func (p *Pipelines) Show(pipeline, mover, next string) (seat.ShowAnswer, error) {
raw, err := p.call(PipelinesShowVerb, seat.ShowInput{Pipeline: pipeline, Mover: mover, Next: next})
if err != nil {
return seat.ShowAnswer{}, fmt.Errorf("%s of %s: %w", PipelinesShowVerb, pipeline, err)
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return seat.ShowAnswer{}, fmt.Errorf("%s of %s answered: %s", PipelinesShowVerb, pipeline, said)
}
var a seat.ShowAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return a, fmt.Errorf("the answer of %s of %s is not a pipeline: %w", PipelinesShowVerb, pipeline, err)
}
return a, nil
}
// followed is the default branch and the watched branches, each once.
func (p *Pipelines) followed() []string {
out := []string{DefaultPipelinesBranch}
if p.branches != nil {
for _, b := range p.branches() {
if b != "" && !slices.Contains(out, b) {
out = append(out, b)
}
}
}
return out
}
// Page is one branch's newest limit pipelines (the default's below one): a followed branch's at the default limit as last
// read, any other branch or limit read now.
func (p *Pipelines) Page(branch string, limit int) PipelinesPage {
if branch = strings.TrimSpace(branch); branch == "" {
branch = DefaultPipelinesBranch
}
if limit < 1 {
limit = DefaultPipelinesLimit
}
if limit == DefaultPipelinesLimit {
p.mu.Lock()
page, ok := p.lists[branch]
p.mu.Unlock()
if ok {
return page
}
}
return p.read(branch, limit)
}
func (p *Pipelines) read(branch string, limit int) PipelinesPage {
rows, err := p.List(branch, limit)
page := PipelinesPage{Branch: branch, Rows: rows}
if err != nil {
page.Error = err.Error()
}
if page.Rows == nil {
page.Rows = []seat.PipelineRow{}
}
return page
}
// Refresh reads the list of every followed branch again and says the lists changed.
func (p *Pipelines) Refresh() {
lists := map[string]PipelinesPage{}
for _, b := range p.followed() {
page := p.read(b, DefaultPipelinesLimit)
if page.Error != "" {
p.logf("the pipelines of %s are not read (read again at the runner's next event): %s", b, page.Error)
}
lists[b] = page
}
p.mu.Lock()
p.lists = lists
p.mu.Unlock()
p.notify("")
}
// Of is the pipeline of a delivery: of the branch's followed list, the newest whose source is the delivery's source
// revision (design 47 H2 H-A-467 (3)); "" when none. A branch whose list is not read yet (a watched branch the reader
// learnt of after its last reading) has the lists read again, once.
func (p *Pipelines) Of(branch string, source uint64) string {
p.mu.Lock()
defer p.mu.Unlock()
page, ok := p.lists[branch]
if !ok {
p.owe()
}
for _, r := range page.Rows { // newest first, as list answers them
if source != 0 && r.Source == int64(source) {
return r.Pipeline
}
}
return ""
}
// Hear takes one event of the runner's (PipelineEvents): the pages of its pipeline are told at once, and the lists are
// read again by Run, which then tells the list pages.
func (p *Pipelines) Hear(data []byte) error {
var ev struct {
Pipeline string `json:"pipeline"`
}
if err := json.Unmarshal(data, &ev); err != nil || ev.Pipeline == "" {
return fmt.Errorf("not an event of a pipeline: %.200s", data)
}
p.notify(ev.Pipeline)
p.owe()
return nil
}
// owe has Run read the lists again.
func (p *Pipelines) owe() {
select {
case p.dirty <- struct{}{}:
default: // a reading is owed already: it reads what this change asks too
}
}
// Run reads the lists at its start and again after each event heard, until ctx ends; events heard while it reads are
// read at once after, in one reading.
func (p *Pipelines) Run(ctx context.Context) {
p.Refresh()
for {
select {
case <-ctx.Done():
return
case <-p.dirty:
p.Refresh()
}
}
}
// notify tells every subscriber what changed: a pipeline's key, or "" for the lists.
func (p *Pipelines) notify(what string) {
p.mu.Lock()
defer p.mu.Unlock()
p.version++
if what != "" {
if _, known := p.heard[what]; !known && len(p.heard) >= p.most {
clear(p.heard)
}
p.heard[what] = p.version
} else {
p.heard[""] = p.version
}
for ch := range p.subs {
select {
case ch <- what:
default: // the subscriber is told already and has not looked yet
}
}
}
// Version is the count of changes said so far, given to a page so its subscription knows what it missed.
func (p *Pipelines) Version() uint64 {
p.mu.Lock()
defer p.mu.Unlock()
return p.version
}
// Changed waits until what (a pipeline's key, or "" for the lists) changes after the version since, or ctx ends: it
// answers at once when it changed already.
func (p *Pipelines) Changed(ctx context.Context, what string, since uint64) bool {
ch := make(chan string, 1)
p.mu.Lock()
if p.heard[what] > since {
p.mu.Unlock()
return true
}
p.subs[ch] = struct{}{}
p.mu.Unlock()
defer func() {
p.mu.Lock()
delete(p.subs, ch)
p.mu.Unlock()
}()
for {
select {
case <-ctx.Done():
return false
case <-ch: // any change: a note dropped while one waited is read from heard, never lost
p.mu.Lock()
changed := p.heard[what] > since
p.mu.Unlock()
if changed {
return true
}
}
}
}
@@ -0,0 +1,298 @@
package view
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// fakeRunner answers the seat mesh-pipeline-runner's list and show as the runner's process does, from rows and pipelines
// the test sets; it counts the lists asked.
type fakeRunner struct {
mu sync.Mutex
rows map[string][]seat.PipelineRow // by branch
shows map[string]seat.ShowAnswer // by pipeline, or pipeline/mover
lists int
}
func (f *fakeRunner) call(key string, body any) (json.RawMessage, error) {
f.mu.Lock()
defer f.mu.Unlock()
switch key {
case PipelinesListVerb:
in := body.(seat.ListInput)
if in.Limit < 1 || in.Full {
return nil, fmt.Errorf("list asked with limit %d, full %v", in.Limit, in.Full)
}
f.lists++
rows := f.rows[in.Branch]
return json.Marshal(seat.ListAnswer{Rows: rows[:min(int(in.Limit), len(rows))]})
case PipelinesShowVerb:
in := body.(seat.ShowInput)
k := in.Pipeline
if in.Mover != "" {
k += "/" + in.Mover
}
a, ok := f.shows[k]
if !ok {
return json.Marshal(map[string]any{"isError": true, "content": []map[string]string{{"type": "text", "text": "no pipeline " + k}}})
}
return json.Marshal(a)
}
return nil, fmt.Errorf("the runner serves no %s", key)
}
func (f *fakeRunner) setRows(branch string, rows ...seat.PipelineRow) {
f.mu.Lock()
defer f.mu.Unlock()
f.rows[branch] = rows
}
// changedWithin is p.Changed bounded, so a change never said fails the test instead of hanging it.
func changedWithin(ctx context.Context, p *Pipelines, what string, since uint64) bool {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return p.Changed(ctx, what, since)
}
// Design 47 H2 H-A-467: the pipelines page reads the runner's list and show, refreshes on its events, never on a timer,
// and links each delivery to its pipeline by its source revision.
func TestThePipelinesPage(t *testing.T) {
started, ended := t0, t0.Add(3*60e9)
runner := &fakeRunner{rows: map[string][]seat.PipelineRow{}, shows: map[string]seat.ShowAnswer{
"main.12": {Pipeline: seat.Pipeline{Pipeline: "main.12", Branch: "main", Source: 9, Revision: 12, Outcome: seat.PipelineOutcomeFailed,
Started: &started, Ended: &ended, Movers: []seat.Mover{{Module: "alpha", Fingerprint: "src2:aa", Version: "0.1.4"}, {Module: "beta", Fingerprint: "src2:bb"}}},
Movers: []seat.MoverRow{{Module: "alpha", Outcome: seat.MoverRowOutcomePassed},
{Module: "beta", Outcome: seat.MoverRowOutcomeFailed, Failed: &seat.FailedStep{Step: "unit", Artifact: "beta:program", Outcome: "could not run", Why: "no build node"}}}},
"main.12/beta": {Pipeline: seat.Pipeline{Pipeline: "main.12", Branch: "main", Source: 9, Revision: 12, Outcome: seat.PipelineOutcomeFailed,
Steps: []seat.StepResult{{Module: "beta", Step: "unit", Artifact: "beta:program", Seat: "node-build-go", Node: "n1", Outcome: "could not run", Started: &started, Ended: &ended, Why: "no build node"}}}},
}}
runner.setRows("main", seat.PipelineRow{Pipeline: "main.12", Source: 9, Revision: 12, Started: &started, Outcome: seat.PipelineOutcomeFailed, MoverCount: 2,
Steps: map[string]int64{"passed": 5, "failed": 1, "could not run": 1, "waiting for a build node": 2}})
runner.setRows(testBranch, seat.PipelineRow{Pipeline: "release.8", Source: 4, Revision: 8, Outcome: seat.PipelineOutcomePassed},
seat.PipelineRow{Pipeline: "release.7", Source: 4, Revision: 7, Outcome: seat.PipelineOutcomeSuperseded})
p := NewPipelines(runner.call, func() []string { return []string{testBranch} }, 0, nil)
m := New(testView)
fill(m, sourceE(t, 4))
m.SetPipelines(p)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go p.Run(ctx)
if !changedWithin(ctx, p, "", 0) { // the first reading at the start
t.Fatal("the lists were never read")
}
web := func(path string) string {
rec := httptest.NewRecorder()
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
if rec.Code != 200 {
t.Fatalf("%s = %d %s", path, rec.Code, rec.Body.String())
}
return rec.Body.String()
}
text := func(f func(*bytes.Buffer)) string {
var b bytes.Buffer
f(&b)
return strings.Join(strings.Fields(b.String()), " ")
}
rows := []struct {
name string
got func() string
want []string
}{
{"the terminal view's list counts the steps per outcome", func() string {
return text(func(b *bytes.Buffer) { PipelinesTable(b, p.Page("", 0)) })
}, []string{"PIPELINES OF main", "PASSED FAILED COULD NOT RUN NOT ASKED WAITING WAITING FOR A BUILD NODE BYPASSED NOTHING TO RUN",
"main.12 12 9 2026-10-18", "failed 2 5 1 1 0 0 2 0 0"}},
{"the web list links each pipeline and reloads on the view's changes, never a timer", func() string { return web("/pipelines") }, []string{
`<a href="/pipelines/show?pipeline=main.12">main.12</a>`, `new EventSource("/pipelines/changes?since=`}},
{"a pipeline's page is its mover rows, each linking its steps", func() string { return web("/pipelines/show?pipeline=main.12") }, []string{
`<a href="/pipelines/show?mover=alpha&amp;pipeline=main.12">alpha</a></td><td>src2:aa</td><td>0.1.4</td><td>passed</td>`,
"<td>unit beta:program: could not run: no build node</td>", `changes?pipeline=main.12\u0026since=`}},
{"a mover's page is its steps", func() string { return web("/pipelines/show?pipeline=main.12&mover=beta") }, []string{
"<td>unit beta:program</td><td>node-build-go</td><td>n1</td><td>could not run</td>", "<td>no build node</td>"}},
{"the terminal view's pipeline names each mover's fingerprint, version and failed step", func() string {
a, err := p.Show("main.12", "", "")
if err != nil {
return err.Error()
}
return text(func(b *bytes.Buffer) { PipelineTable(b, a, "") })
}, []string{"PIPELINE main.12: failed, source 9, revision 12", "alpha src2:aa 0.1.4 passed", "beta src2:bb failed unit beta:program: could not run: no build node"}},
{"the web list asks the newest limit of a branch", func() string { return web("/pipelines?branch=" + testBranch + "&limit=1") }, []string{
"release.8", `name="limit" value="1"`}},
{"the web list is the newest 20 by default", func() string { return web("/pipelines?branch=" + testBranch) }, []string{
"release.8", "release.7", `name="limit" value="20"`}},
{"a pipeline the runner does not hold says so", func() string { return web("/pipelines/show?pipeline=main.99") }, []string{"unreadable: ", "no pipeline main.99"}},
{"a delivery links to the newest pipeline of its source revision", func() string {
return text(func(b *bytes.Buffer) { Table(b, m.Snapshot()) }) + web("/")
}, []string{testBranch + ": source revision 4: delivered (pipeline release.8)", `(<a href="/pipelines/show?pipeline=release.8">pipeline</a>)`, `<a href="/pipelines">Pipelines</a>`}},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
got := r.got()
for _, w := range r.want {
if !strings.Contains(got, w) {
t.Errorf("got %s\nwant it to hold %q", got, w)
}
}
})
}
if got := web("/pipelines?branch=" + testBranch + "&limit=1"); strings.Contains(got, "release.7") {
t.Errorf("the newest 1 shows release.7: %s", got)
}
for _, l := range []string{"0", "x"} {
rec := httptest.NewRecorder()
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/pipelines?limit="+l, nil))
if rec.Code != http.StatusBadRequest {
t.Errorf("limit %s = %d, want 400", l, rec.Code)
}
}
if strings.Contains(web("/pipelines"), "http-equiv=\"refresh\"") || strings.Contains(web("/pipelines/show?pipeline=main.12"), "http-equiv=\"refresh\"") {
t.Error("a pipelines page reloads on a timer")
}
t.Run("an event of the runner reads the lists again and tells the pages", func(t *testing.T) {
srv := httptest.NewServer(Web(m))
defer srv.Close()
v := p.Version()
res, err := http.Get(srv.URL + "/pipelines/changes?since=" + fmt.Sprint(v))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
stream := bufio.NewReader(res.Body)
if line, _ := stream.ReadString('\n'); !strings.HasPrefix(line, ":") {
t.Fatalf("the stream starts %q", line)
}
shown := make(chan bool, 1)
go func() { shown <- changedWithin(ctx, p, "main.13", v) }()
runner.mu.Lock()
before := runner.lists
runner.mu.Unlock()
runner.setRows("main", seat.PipelineRow{Pipeline: "main.13", Source: 10, Revision: 13, Outcome: seat.PipelineOutcomeRunning, MoverCount: 1})
if err := p.Hear([]byte(`{"pipeline":"main.13","branch":"main","source":10}`)); err != nil {
t.Fatal(err)
}
if !<-shown {
t.Fatal("the page of the pipeline heard is not told")
}
for {
line, err := stream.ReadString('\n')
if err != nil {
t.Fatalf("the stream ended before the change: %v", err)
}
if line == "data: changed\n" {
break
}
}
runner.mu.Lock()
after := runner.lists
runner.mu.Unlock()
if !strings.Contains(web("/pipelines"), "main.13") || after <= before {
t.Errorf("the list was not read again (%d lists, %d before): %s", after, before, web("/pipelines"))
}
if err := p.Hear([]byte(`{"step":"unit"}`)); err == nil {
t.Error("an event naming no pipeline is taken")
}
})
t.Run("the terminal view follows the list on the socket, again at each event", func(t *testing.T) {
path, sm := served(t)
sm.SetPipelines(p)
var seen []string
heard := false
err := AskPipelines(path, "", 0, "", "", "", true, func(page PipelinesPage, _ *seat.ShowAnswer) bool {
seen = append(seen, page.Rows[0].Pipeline)
if !heard {
heard = true
runner.setRows("main", seat.PipelineRow{Pipeline: "main.14", Source: 11, Revision: 14, Outcome: seat.PipelineOutcomeRunning, MoverCount: 1})
if err := p.Hear([]byte(`{"pipeline":"main.14","mover":"alpha","revision":14}`)); err != nil { // a re-pinned names no branch
t.Error(err)
}
return true
}
return len(seen) < 2 || seen[len(seen)-1] != "main.14"
})
if err != nil || seen[len(seen)-1] != "main.14" {
t.Fatalf("followed %v, %v", seen, err)
}
var show *seat.ShowAnswer
if err := AskPipelines(path, "", 0, "main.12", "beta", "", false, func(_ PipelinesPage, a *seat.ShowAnswer) bool { show = a; return true }); err != nil {
t.Fatal(err)
}
if len(show.Pipeline.Steps) != 1 || show.Pipeline.Steps[0].Node != "n1" {
t.Fatalf("the mover's steps on the socket = %+v", show)
}
if err := AskPipelines(path, "", 0, "main.99", "", "", false, func(PipelinesPage, *seat.ShowAnswer) bool { return true }); err == nil || !strings.Contains(err.Error(), "no pipeline main.99") {
t.Fatalf("a pipeline the runner does not hold = %v", err)
}
})
t.Run("the pipelines heard are kept up to the setting pipelines-heard", func(t *testing.T) {
few := NewPipelines(runner.call, nil, 2, nil)
for _, k := range []string{"main.1", "main.2", "main.2", "main.3"} {
few.notify(k)
}
few.mu.Lock()
n, last := len(few.heard), few.heard["main.3"]
few.mu.Unlock()
if n > 2 || last == 0 {
t.Errorf("heard holds %d pipelines (main.3 at %d), want at most the setting's 2, the newest kept", n, last)
}
if d := NewPipelines(runner.call, nil, 0, nil); d.most != DefaultPipelinesHeard {
t.Errorf("the default keeps %d, want %d", d.most, DefaultPipelinesHeard)
}
})
t.Run("a list not read is shown unreadable and followed on, read again at the next event", func(t *testing.T) {
var mu sync.Mutex
away := true // the runner not answering yet, as the bus says when no holder listens
call := func(key string, body any) (json.RawMessage, error) {
mu.Lock()
defer mu.Unlock()
if away {
return nil, fmt.Errorf("nats: no responders available for request")
}
return runner.call(key, body)
}
up := NewPipelines(call, nil, 0, nil)
upCtx, upCancel := context.WithCancel(ctx)
defer upCancel()
go up.Run(upCtx)
if !changedWithin(upCtx, up, "", 0) {
t.Fatal("the lists were never read")
}
path, sm := served(t)
sm.SetPipelines(up)
var seen []PipelinesPage
err := AskPipelines(path, "", 0, "", "", "", true, func(page PipelinesPage, _ *seat.ShowAnswer) bool {
seen = append(seen, page)
if len(seen) == 1 {
mu.Lock()
away = false
mu.Unlock()
if err := up.Hear([]byte(`{"pipeline":"main.14"}`)); err != nil {
t.Error(err)
}
}
return page.Error != "" || len(page.Rows) == 0
})
if err != nil || len(seen) < 2 || !strings.Contains(seen[0].Error, "no responders") || seen[len(seen)-1].Rows[0].Pipeline != "main.14" {
t.Fatalf("followed %+v, %v; want the unreadable list, then main.14 after the event", seen, err)
}
var b bytes.Buffer
PipelinesTable(&b, seen[0])
if !strings.Contains(b.String(), "unreadable: ") {
t.Errorf("the terminal view of an unreadable list = %q", b.String())
}
})
}
@@ -0,0 +1,286 @@
package view
import (
"fmt"
"html/template"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"text/tabwriter"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// PipelineCountOutcomes are the step outcomes a list row counts, in the page's order (design 47 H2 H-A-467 (2)).
var PipelineCountOutcomes = []string{seat.StepResultOutcomePassed, seat.StepResultOutcomeFailed, seat.StepResultOutcomeCouldNotRun,
seat.StepResultOutcomeNotAsked, seat.StepResultOutcomeWaiting, seat.StepResultOutcomeWaitingForABuildNode,
seat.StepResultOutcomeBypassed, seat.StepResultOutcomeNothingToRun}
// PipelinesHead is the head of the list's rows.
var PipelinesHead = append([]string{"PIPELINE", "REVISION", "SOURCE", "STARTED", "OUTCOME", "MOVERS"}, upper(PipelineCountOutcomes)...)
// MoversHead is the head of one pipeline's mover rows, StepsHead of one mover's steps (design 47 H2 H-A-467 (2)).
var (
MoversHead = []string{"MOVER", "FINGERPRINT", "VERSION", "OUTCOME", "FAILED STEP"}
StepsHead = []string{"STEP", "SEAT", "NODE", "OUTCOME", "STARTED", "ENDED", "WHY"}
)
func upper(in []string) []string {
out := make([]string, len(in))
for i, s := range in {
out[i] = strings.ToUpper(s)
}
return out
}
func timeOf(t *time.Time) string {
if t == nil {
return ""
}
return since(*t)
}
func number(n int64) string {
if n == 0 {
return ""
}
return strconv.FormatInt(n, 10)
}
// PipelineRowText is one list row's cells, in PipelinesHead's order.
func PipelineRowText(r seat.PipelineRow) []string {
out := []string{r.Pipeline, number(r.Revision), number(r.Source), timeOf(r.Started), r.Outcome, strconv.FormatInt(r.MoverCount, 10)}
for _, o := range PipelineCountOutcomes {
out = append(out, strconv.FormatInt(r.Steps[o], 10))
}
return out
}
func failedText(f *seat.FailedStep) string {
if f == nil {
return ""
}
s := f.Step
if f.Artifact != "" {
s += " " + f.Artifact
}
s += ": " + f.Outcome
if f.Why != "" {
s += ": " + f.Why
}
return s
}
// MoverRowsText is one pipeline's mover rows as show without a mover answers them: each row's module, outcome and failed
// step, beside the pipeline's mover of that module's fingerprint and published version.
func MoverRowsText(a seat.ShowAnswer) [][]string {
movers := map[string]seat.Mover{}
for _, m := range a.Pipeline.Movers {
movers[m.Module] = m
}
var out [][]string
for _, r := range a.Movers {
m := movers[r.Module]
out = append(out, []string{r.Module, m.Fingerprint, m.Version, r.Outcome, failedText(r.Failed)})
}
return out
}
// StepsText is one mover's steps as show with the mover answers them.
func StepsText(a seat.ShowAnswer) [][]string {
var out [][]string
for _, s := range a.Pipeline.Steps {
step := s.Step
if s.Artifact != "" {
step += " " + s.Artifact
}
out = append(out, []string{step, s.Seat, s.Node, s.Outcome, timeOf(s.Started), timeOf(s.Ended), s.Why})
}
return out
}
func table(w io.Writer, head []string, rows [][]string) {
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, strings.Join(head, "\t"))
for _, r := range rows {
fmt.Fprintln(tw, strings.Join(r, "\t"))
}
tw.Flush()
}
// PipelinesTable prints one branch's newest pipelines, the terminal view's pipelines page.
func PipelinesTable(w io.Writer, p PipelinesPage) {
fmt.Fprintf(w, "PIPELINES OF %s\n", p.Branch)
if p.Error != "" {
fmt.Fprintf(w, "unreadable: %s\n", p.Error)
return
}
var rows [][]string
for _, r := range p.Rows {
rows = append(rows, PipelineRowText(r))
}
table(w, PipelinesHead, rows)
}
// PipelineTable prints one pipeline as show answered it for mover: its mover rows, or that mover's steps.
func PipelineTable(w io.Writer, a seat.ShowAnswer, mover string) {
p := a.Pipeline
fmt.Fprintf(w, "PIPELINE %s: %s, source %d, revision %d, started %s, ended %s\n", p.Pipeline, p.Outcome, p.Source, p.Revision, timeOf(p.Started), timeOf(p.Ended))
if p.Why != "" {
fmt.Fprintf(w, "why: %s\n", p.Why)
}
if mover == "" {
table(w, MoversHead, MoverRowsText(a))
} else {
fmt.Fprintf(w, "STEPS OF %s\n", mover)
table(w, StepsHead, StepsText(a))
}
if a.Next != "" {
fmt.Fprintf(w, "more: -next %s\n", a.Next)
}
}
const pipelinesStyle = `<style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style>`
// changesScript reloads the page when the view says its part changed, on the runner's events (design 47 H2 H-A-467): never
// on a timer.
const changesScript = `<script>new EventSource({{.Changes}}).onmessage=function(){location.reload()}</script>`
var pipelinesPage = template.Must(template.New("pl").Parse(`<!doctype html><html><head><meta charset="utf-8">
<title>Pipelines of {{.Branch}}</title>` + pipelinesStyle + `</head>
<body><p><a href="/">What each node runs</a></p><h1>PIPELINES OF {{.Branch}}</h1>
<form method="get" action="/pipelines">branch <input name="branch" value="{{.Branch}}"> newest <input name="limit" value="{{.Limit}}" size="4"> <button>show</button></form>
{{if .Error}}<p>unreadable: {{.Error}}</p>{{else}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
{{range .Rows}}<tr><td><a href="{{.Link}}">{{index .Cells 0}}</a></td>{{range slice .Cells 1}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
` + changesScript + `</body></html>`))
var pipelinePage = template.Must(template.New("p1").Parse(`<!doctype html><html><head><meta charset="utf-8">
<title>Pipeline {{.Pipeline}}</title>` + pipelinesStyle + `</head>
<body><p><a href="/">What each node runs</a> · <a href="{{.Back}}">Pipelines of {{.Branch}}</a>{{if .Mover}} · <a href="{{.Up}}">Pipeline {{.Pipeline}}</a>{{end}}</p>
<h1>PIPELINE {{.Pipeline}}{{if .Mover}}: {{.Mover}}{{end}}</h1>
{{if .Error}}<p>unreadable: {{.Error}}</p>{{else}}<p>{{.Line}}</p>{{if .Why}}<p>why: {{.Why}}</p>{{end}}
<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
{{range .Rows}}<tr>{{if .Link}}<td><a href="{{.Link}}">{{index .Cells 0}}</a></td>{{else}}<td>{{index .Cells 0}}</td>{{end}}{{range slice .Cells 1}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
{{if .More}}<p><a href="{{.More}}">more</a></p>{{end}}{{end}}
` + changesScript + `</body></html>`))
type linkedRow struct {
Cells []string
Link string
}
// PipelineLink is the page of one pipeline, or of one of its movers.
func PipelineLink(pipeline, mover string) string {
q := url.Values{"pipeline": {pipeline}}
if mover != "" {
q.Set("mover", mover)
}
return "/pipelines/show?" + q.Encode()
}
func changesLink(pipeline string, version uint64) string {
q := url.Values{"since": {strconv.FormatUint(version, 10)}}
if pipeline != "" {
q.Set("pipeline", pipeline)
}
return "/pipelines/changes?" + q.Encode()
}
// webPipelines serves the pipelines pages (design 47 H2 H-A-467): the list of a branch, one pipeline's mover rows, one
// mover's steps, and the changes each page reloads on.
func webPipelines(m *Model, w http.ResponseWriter, r *http.Request) {
p := m.Pipelines()
if p == nil {
http.Error(w, "the view reads no pipelines yet", http.StatusServiceUnavailable)
return
}
q := r.URL.Query()
switch r.URL.Path {
case "/pipelines/changes":
changes(p, w, r, q.Get("pipeline"))
case "/pipelines/show":
key, mover := q.Get("pipeline"), q.Get("mover")
if key == "" {
http.Error(w, "name a pipeline", http.StatusBadRequest)
return
}
v := p.Version()
a, err := p.Show(key, mover, q.Get("next"))
data := map[string]any{"Pipeline": key, "Mover": mover, "Changes": changesLink(key, v), "Up": PipelineLink(key, "")}
if err != nil {
data["Error"] = err.Error()
} else {
pl := a.Pipeline
data["Branch"] = pl.Branch
data["Back"] = "/pipelines?" + url.Values{"branch": {pl.Branch}}.Encode()
data["Line"] = fmt.Sprintf("%s, source %d, revision %d, started %s, ended %s", pl.Outcome, pl.Source, pl.Revision, timeOf(pl.Started), timeOf(pl.Ended))
data["Why"] = pl.Why
var rows []linkedRow
if mover == "" {
data["Head"] = MoversHead
for _, c := range MoverRowsText(a) {
rows = append(rows, linkedRow{Cells: c, Link: PipelineLink(key, c[0])})
}
} else {
data["Head"] = StepsHead
for _, c := range StepsText(a) {
rows = append(rows, linkedRow{Cells: c})
}
}
data["Rows"] = rows
if a.Next != "" {
more := url.Values{"pipeline": {key}, "next": {a.Next}}
if mover != "" {
more.Set("mover", mover)
}
data["More"] = "/pipelines/show?" + more.Encode()
}
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
pipelinePage.Execute(w, data)
default:
limit := DefaultPipelinesLimit
if l := strings.TrimSpace(q.Get("limit")); l != "" {
n, err := strconv.Atoi(l)
if err != nil || n < 1 {
http.Error(w, "limit is a number of at least one", http.StatusBadRequest)
return
}
limit = n
}
v := p.Version()
page := p.Page(q.Get("branch"), limit)
var rows []linkedRow
for _, row := range page.Rows {
rows = append(rows, linkedRow{Cells: PipelineRowText(row), Link: PipelineLink(row.Pipeline, "")})
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
pipelinesPage.Execute(w, map[string]any{"Branch": page.Branch, "Error": page.Error, "Head": PipelinesHead, "Rows": rows, "Limit": limit, "Changes": changesLink("", v)})
}
}
// changes is a stream of server-sent events: one message each time the page's part changed (a pipeline's key, or "" for
// the lists) after the version the page was made at, until the browser leaves.
func changes(p *Pipelines, w http.ResponseWriter, r *http.Request, what string) {
fl, ok := w.(http.Flusher)
if !ok {
http.Error(w, "this server cannot stream", http.StatusInternalServerError)
return
}
since, _ := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64)
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, ": the page reloads at the runner's next event\n\n")
fl.Flush()
for {
if !p.Changed(r.Context(), what, since) {
return
}
since = p.Version()
fmt.Fprint(w, "data: changed\n\n")
fl.Flush()
}
}
@@ -15,6 +15,8 @@ import (
"strings"
"text/tabwriter"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// ListenSocket listens on the unix socket at path, in a directory of mode 0755, with mode 0666 on the socket: the
@@ -48,6 +50,14 @@ type ask struct {
Words string `json:"words,omitempty"`
Module string `json:"module,omitempty"`
FP string `json:"fp,omitempty"`
// The pipelines page's asks (design 47 H2 H-A-467): pipelines of a branch, or one pipeline, of a mover when named; follow
// keeps the connection and answers again at each change the runner's events say.
Branch string `json:"branch,omitempty"`
Limit int `json:"limit,omitempty"` // the newest this many of the branch, the default's when left out
Pipeline string `json:"pipeline,omitempty"`
Mover string `json:"mover,omitempty"`
Next string `json:"next,omitempty"`
Follow bool `json:"follow,omitempty"`
}
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
@@ -74,11 +84,14 @@ func answer(c net.Conn, m *Model) {
return
}
var a ask
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer" && a.Ask != "holds" && a.Ask != "hold") {
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"}, {"ask":"answer"}, {"ask":"holds"} and {"ask":"hold"} and nothing else`})
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer" && a.Ask != "holds" && a.Ask != "hold" && a.Ask != "pipelines" && a.Ask != "pipeline") {
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"}, {"ask":"answer"}, {"ask":"holds"}, {"ask":"hold"}, {"ask":"pipelines"} and {"ask":"pipeline"} and nothing else`})
return
}
switch a.Ask {
case "pipelines", "pipeline":
answerPipelines(c, m, a)
return
case "holds":
json.NewEncoder(c).Encode(m.Holds())
return
@@ -253,12 +266,38 @@ func wantText(s Snapshot) [][]string {
func sourceText(s Snapshot) []string {
var out []string
for _, src := range s.Sources {
out = append(out, fmt.Sprintf("%s: source revision %d: %s", src.Branch, src.Revision, src.State))
line := fmt.Sprintf("%s: source revision %d: %s", src.Branch, src.Revision, src.State)
if src.Pipeline != "" { // its pipeline (design 47 H2 H-A-467 (3))
line += " (pipeline " + src.Pipeline + ")"
}
out = append(out, line)
out = append(out, src.Waiting...)
}
return out
}
// sourceLine is one line of the source revisions on the page, with the link to its pipeline when it has one.
type sourceLine struct {
Line string
Link string
}
// sourceLines are sourceText's lines on the page: each delivery's line links to its pipeline (design 47 H2 H-A-467 (3)).
func sourceLines(s Snapshot) []sourceLine {
var out []sourceLine
for _, src := range s.Sources {
l := sourceLine{Line: fmt.Sprintf("%s: source revision %d: %s", src.Branch, src.Revision, src.State)}
if src.Pipeline != "" {
l.Link = PipelineLink(src.Pipeline, "")
}
out = append(out, l)
for _, w := range src.Waiting {
out = append(out, sourceLine{Line: w})
}
}
return out
}
// backlogText is one line per language and kind: its waiting asks, the asks held by a node, the oldest waiting ask's age.
func backlogText(s Snapshot) [][]string {
var out [][]string
@@ -380,12 +419,12 @@ func (o OfflineRow) name() string {
var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><meta charset="utf-8"><meta http-equiv="refresh" content="5">
<title>Delivery</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
<body><h1>What each node runs</h1><table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
<body><p><a href="/pipelines">Pipelines</a></p><h1>What each node runs</h1><table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
{{range .Rows}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Asks}}<h2>ASKS</h2><table><tr><th>ASK</th><th>HEADLINE</th><th>WORDS</th><th>PULL REQUESTS</th><th>ANSWER</th></tr>{{range .Asks}}<tr>{{range .Cells}}<td>{{.}}</td>{{end}}<td>{{if .Options}}<form method="post" action="/answer"><input type="hidden" name="key" value="{{.Key}}"><input name="words" placeholder="words (optional)">{{range .Options}} <button name="option" value="{{.}}">{{.}}</button>{{end}}</form>{{end}}</td></tr>{{end}}</table>{{end}}
{{if .Backlog}}<h2>BACKLOG OF BUILD ASKS</h2><table><tr>{{range .BacklogHead}}<th>{{.}}</th>{{end}}</tr>{{range .Backlog}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Source}}<h2>SOURCE REVISION</h2><p>{{range .Source}}{{.}}<br>{{end}}</p>{{end}}{{if .Wants}}<h2>WANTS AGAINST HAVES</h2><table><tr>{{range .WantHead}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Source}}<h2>SOURCE REVISION</h2><p>{{range .Source}}{{.Line}}{{if .Link}} (<a href="{{.Link}}">pipeline</a>){{end}}<br>{{end}}</p>{{end}}{{if .Wants}}<h2>WANTS AGAINST HAVES</h2><table><tr>{{range .WantHead}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
@@ -414,6 +453,10 @@ func Web(m *Model) http.Handler {
http.Error(w, "this view is read-only", http.StatusMethodNotAllowed)
return
}
if r.URL.Path == "/pipelines" || strings.HasPrefix(r.URL.Path, "/pipelines/") {
webPipelines(m, w, r)
return
}
if r.URL.Path == "/wants" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
wantsPage.Execute(w, map[string]any{"Head": wantHead, "Wants": wantText(m.Snapshot())})
@@ -439,7 +482,7 @@ func Web(m *Model) http.Handler {
for _, u := range snap.Unreadable {
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
}
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceLines(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
})
}
@@ -498,3 +541,101 @@ func answerForm(m *Model, w http.ResponseWriter, r *http.Request) {
}
http.Redirect(w, r, "/", http.StatusSeeOther)
}
// answerPipelines answers the pipelines page's asks: a branch's list, or one pipeline as show answers it; with follow, again
// at each change of its part the runner's events say (design 47 H2 H-A-467), until the asker leaves.
func answerPipelines(c net.Conn, m *Model, a ask) {
p := m.Pipelines()
enc := json.NewEncoder(c)
if p == nil {
enc.Encode(pipelinesRefusal{Refused: "the view reads no pipelines yet"})
return
}
if a.Ask == "pipeline" && a.Pipeline == "" {
enc.Encode(pipelinesRefusal{Refused: "name a pipeline"})
return
}
what := ""
if a.Ask == "pipeline" {
what = a.Pipeline
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if a.Follow {
c.SetDeadline(time.Time{})
go func() { // the asker leaving ends the following
io.Copy(io.Discard, c)
cancel()
}()
}
for {
v := p.Version()
var err error
if a.Ask == "pipelines" {
err = enc.Encode(p.Page(a.Branch, a.Limit))
} else if sa, serr := p.Show(a.Pipeline, a.Mover, a.Next); serr != nil {
err = enc.Encode(pipelinesRefusal{Refused: serr.Error()})
} else {
err = enc.Encode(sa)
}
if err != nil || !a.Follow || !p.Changed(ctx, what, v) {
return
}
}
}
// pipelinesRefusal is the socket's answer to a pipelines ask it cannot answer: no reader, no pipeline named, or a pipeline
// show could not read. A list that could not be read is no refusal: it is a page whose Error says why, handed on, and a
// following reads it again at the runner's next event (design 47 H2 H-A-467).
type pipelinesRefusal struct {
Refused string `json:"refused"`
}
// AskPipelines asks the socket at path for one branch's newest limit pipelines (the default's below one), or with pipeline
// set for that pipeline (of mover when named); with follow, each answers again at each change, handed to each until it
// answers false or the view leaves.
func AskPipelines(path, branch string, limit int, pipeline, mover, next string, follow bool, each func(PipelinesPage, *seat.ShowAnswer) bool) error {
c, err := net.DialTimeout("unix", path, 5*time.Second)
if err != nil {
return fmt.Errorf("cannot reach the view at %s: %w", path, err)
}
defer c.Close()
a := ask{Ask: "pipelines", Branch: branch, Limit: limit, Follow: follow}
if pipeline != "" {
a = ask{Ask: "pipeline", Pipeline: pipeline, Mover: mover, Next: next, Follow: follow}
}
if !follow {
c.SetDeadline(time.Now().Add(10 * time.Second))
}
req, _ := json.Marshal(a)
if _, err := c.Write(append(req, '\n')); err != nil {
return err
}
dec := json.NewDecoder(c)
for {
var raw json.RawMessage
if err := dec.Decode(&raw); err != nil {
if errors.Is(err, io.EOF) && follow {
return errors.New("the view ended the following")
}
return fmt.Errorf("the view's answer is not pipelines: %w", err)
}
var refused pipelinesRefusal
if json.Unmarshal(raw, &refused) == nil && refused.Refused != "" {
return errors.New(refused.Refused)
}
var page PipelinesPage
var show *seat.ShowAnswer
if pipeline != "" {
show = &seat.ShowAnswer{}
if err := json.Unmarshal(raw, show); err != nil {
return fmt.Errorf("the view's answer is not a pipeline: %w", err)
}
} else if err := json.Unmarshal(raw, &page); err != nil {
return fmt.Errorf("the view's answer is not pipelines: %w", err)
}
if !each(page, show) || !follow {
return nil
}
}
}
@@ -145,6 +145,9 @@ type SourceRow struct {
// Away are the nodes the watchman says are offline, left out of the comparison (ADR 0603 §7): the revision counts for them
// once they return and run their wants.
Away []string `json:"away,omitempty"`
// Pipeline is the pipeline of the delivery: the newest of the branch whose source is this source revision, as the seat
// mesh-pipeline-runner's list answers it (design 47 H2 H-A-467 (3)); empty when none is listed.
Pipeline string `json:"pipeline,omitempty"`
}
// AskRow is an open ask (novox/hq ADR 0437 §10): the gate's ask with no answer under its key in the teller's answers.
@@ -271,6 +274,7 @@ type Model struct {
reachSaid map[string]uint64 // the reached head's source revision said last, by repository key
reachNotYet map[string]string // the merges said not yet on the machines with it, by repository key
reachPut func(ReachWrite) error
pipelines *Pipelines // the runner's pipelines, read through its seat's verbs (design 47 H2 H-A-467)
}
type presenceEntry struct {
@@ -710,15 +714,34 @@ func errText(err error) string {
func (m *Model) Snapshot() Snapshot {
s := m.snapshot()
m.mu.Lock()
b := m.backlog
b, p := m.backlog, m.pipelines
m.mu.Unlock()
s.Backlog = []BacklogRow{}
if b != nil {
s.Backlog = b.Rows() // read from the bus outside the model's lock
}
if p != nil {
for i, src := range s.Sources { // each delivery links to its pipeline (design 47 H2 H-A-467 (3))
s.Sources[i].Pipeline = p.Of(src.Branch, src.Revision)
}
}
return s
}
// SetPipelines gives the model the runner's pipelines, which the pipelines page shows and each delivery links to.
func (m *Model) SetPipelines(p *Pipelines) {
m.mu.Lock()
defer m.mu.Unlock()
m.pipelines = p
}
// Pipelines is the reader SetPipelines gave, nil before.
func (m *Model) Pipelines() *Pipelines {
m.mu.Lock()
defer m.mu.Unlock()
return m.pipelines
}
// SetBacklog gives the model the seats' workers to read the backlog of asks from.
// wantsHeld is the real bucket of mesh-wants' wants at the holder followed now, "" for none.
func (m *Model) wantsHeld() string {
+42 -22
View File
@@ -71,26 +71,33 @@ type claim struct {
}
type manifest struct {
Packages map[string]string `json:"packages"`
Claims []claim `json:"claims"`
Class string `json:"class"`
State []state `json:"state"`
Module string `json:"module"`
Moved bool `json:"moved"`
Version string `json:"version"`
Slug string `json:"slug"`
Reads []string `json:"reads"`
Consumes []string `json:"consumes"`
Invokes []invoke `json:"invokes"`
Uses []string `json:"uses"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Streams []stream `json:"streams"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
Requires []require `json:"requires"`
Items []item `json:"items"`
Packages map[string]string `json:"packages"`
Claims []claim `json:"claims"`
Class string `json:"class"`
State []state `json:"state"`
Module string `json:"module"`
Moved bool `json:"moved"`
Version string `json:"version"`
Slug string `json:"slug"`
Reads []string `json:"reads"`
Consumes []string `json:"consumes"`
Invokes []invoke `json:"invokes"`
Uses []string `json:"uses"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Streams []stream `json:"streams"`
OwnSecrets map[string]string `json:"own-secrets"`
Config map[string]struct {
Type string `json:"type"`
Description string `json:"description"`
Trusted bool `json:"trusted"`
Minimum float64 `json:"minimum"`
Default any `json:"default"`
} `json:"config"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
Requires []require `json:"requires"`
Items []item `json:"items"`
Build struct {
Artifacts []artifact `json:"artifacts"`
} `json:"build"`
@@ -186,7 +193,8 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
{Verb: "seat:mesh-wants.revision"}, // §512, A-242: view_tiers compares kept assignment revisions
{Verb: "seat:mesh-wants.difference"}, // §512, A-242: the tiers and each change's after are the assigner's
{Verb: "seat:node-build-go.current"}, {Verb: "seat:node-build-image.current"}, {Verb: "seat:node-build-node.current"}, // design 47 H2 §813 (9): build_usage reads each holder's work by its seat's verb current
{Verb: "seat:node-engine.manifest-reads"}}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2)
{Verb: "seat:node-engine.manifest-reads"},
{Verb: "seat:mesh-pipeline-runner.list"}, {Verb: "seat:mesh-pipeline-runner.show"}}; !reflect.DeepEqual(m.Invokes, want) { // design 47 H2 H-A-467: the pipelines page reads the runner's list and show // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2)
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// design 47 G §387: the keeper's ask for the findings is heard, the finding stream declared (§383), and §368: version is MAJOR.MINOR.
@@ -207,7 +215,8 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.taken", "mesh-delivery-teller.views.answers",
"mesh-wants.wait-held", "mesh-wants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended",
"mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended",
"mesh-broker.recreated", "mesh-broker.dropped"}; !reflect.DeepEqual(m.Consumes, want) { // design 47 G §512, A-242: every part's wait-held and wait-ended
"mesh-broker.recreated", "mesh-broker.dropped",
"mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.re-pinned"}; !reflect.DeepEqual(m.Consumes, want) { // design 47 G §512, A-242: every part's wait-held and wait-ended; design 47 H2 H-A-467: the runner's events the pipelines page reads again on
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
@@ -385,6 +394,17 @@ func TestManifestResources(t *testing.T) { // 1134/AC7
if env["MESH_BROKER_FILE"] != "${dir:state}/broker" || m.OwnSecrets["broker"] != "${dir:state}/broker" {
t.Errorf("the process reads %v, the own-secret is written at %v", env["MESH_BROKER_FILE"], m.OwnSecrets["broker"])
}
// The setting pipelines-heard reaches the process as a file of its state directory, read at its start (principle 11).
if c, ok := m.Config["pipelines-heard"]; !ok || c.Default != float64(view.DefaultPipelinesHeard) || c.Minimum != 1 || c.Type != "number" || len(m.Config) != 1 {
t.Errorf("config = %+v, want pipelines-heard, a number of at least 1, %d by default", m.Config, view.DefaultPipelinesHeard)
}
var heardFile bool
for _, r := range byType["file"] {
heardFile = heardFile || (r["id"] == "pipelines-heard" && r["path"] == "${dir:state}/pipelines-heard" && r["content"] == "${config:pipelines-heard}" && r["owner"] == user && r["mode"] == "0600")
}
if !heardFile || env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"] != "${dir:state}/pipelines-heard" {
t.Errorf("the setting pipelines-heard is written to its file %v, the process reads %v", heardFile, env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"])
}
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" {
t.Errorf("health = %v", p["health"])
}
+27 -3
View File
@@ -102,7 +102,11 @@
"mesh-delivery-builder.wait-held",
"mesh-delivery-builder.wait-ended",
"mesh-broker.recreated",
"mesh-broker.dropped"
"mesh-broker.dropped",
"mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.step-ended",
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.re-pinned"
],
"invokes": [
"mesh-delivery-assigner.assigner_read_differences",
@@ -128,7 +132,9 @@
"seat:node-build-go.current",
"seat:node-build-image.current",
"seat:node-build-node.current",
"seat:node-engine.manifest-reads"
"seat:node-engine.manifest-reads",
"seat:mesh-pipeline-runner.list",
"seat:mesh-pipeline-runner.show"
],
"reads": [
"seat:node-build-go.pauses",
@@ -148,6 +154,15 @@
"seat:mesh-vault.signers",
"node-engine.have"
],
"config": {
"pipelines-heard": {
"type": "number",
"description": "How many pipelines' last changes the view keeps for its pipelines pages before it forgets them all, read at its start (design 47 H2 H-A-467; principle 11).",
"trusted": false,
"minimum": 1,
"default": 1024
}
},
"own-secrets": {
"broker": "${dir:state}/broker"
},
@@ -185,13 +200,22 @@
"./mesh-delivery-view"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker"
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE": "${dir:state}/pipelines-heard"
},
"user": "mesh-delivery-view",
"health": {
"kind": "unit"
}
},
{
"id": "pipelines-heard",
"type": "file",
"path": "${dir:state}/pipelines-heard",
"owner": "mesh-delivery-view",
"mode": "0600",
"content": "${config:pipelines-heard}"
},
{
"id": "command-program",
"type": "archive",