diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go index d36a6814..567357fd 100644 --- a/cmd/mesh-controller/queue.go +++ b/cmd/mesh-controller/queue.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "regexp" "sort" "strings" "time" @@ -67,18 +68,95 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error { if err != nil { return err } + now := time.Now() if *asJSON { - body, err := json.MarshalIndent(q, "", " ") + body, err := json.MarshalIndent(queueAnswerOf(q, now), "", " ") if err != nil { return err } fmt.Fprintln(w, string(body)) return nil } - fmt.Fprint(w, queueText(q, time.Now())) + fmt.Fprint(w, queueText(q, now)) return nil } +// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask +// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the +// table, written from the same read: the verb answers it as its output and the rest as its answer, so the +// two never disagree. +type queueAnswer struct { + Seat string `json:"seat"` + Waiting int `json:"waiting"` + InFlight int `json:"in-flight"` + Dead int `json:"dead"` + MaxDeliver int `json:"max-deliver"` + Asks []queuedAnswer `json:"asks"` + Text string `json:"text"` +} + +// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every +// artifact the mesh has built, nor a check's checkouts beside it. +type queuedAnswer struct { + ID string `json:"id"` + State string `json:"state"` + // Kind is "check" for a pull request's merge check, "build" for anything else. + Kind string `json:"kind"` + Repository string `json:"repository"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + // Commit is the ref when it names a commit; empty when the ask follows a branch or the default. + Commit string `json:"commit,omitempty"` + // Machine is the one building an ask in flight; WasOn the one that started it and handed it back. + Machine string `json:"machine,omitempty"` + WasOn string `json:"was-on,omitempty"` + AskedAt time.Time `json:"asked-at,omitzero"` + // Since is when the ask came to its state: its start on the machine for one in flight that a + // machine said it started, otherwise when it was asked. + Since time.Time `json:"since,omitzero"` + // Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting. + Position int `json:"position,omitempty"` + Seq uint64 `json:"seq"` +} + +// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256). +var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`) + +// queueAnswerOf is the queue as data, in the table's order: in flight, the waiting from the next one to +// be handed out, then the dead. A pure function of one read (novox/hq issue 455). +func queueAnswerOf(q link.Queue, now time.Time) queueAnswer { + waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead) + out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead), + MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)} + one := func(a link.QueuedAsk, position int) queuedAnswer { + kind := "build" + if a.Request.Check != nil { + kind = "check" + } + since := a.AskedAt + if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() { + since = a.Started + } + commit := "" + if aCommitRef.MatchString(a.Ref) { + commit = a.Ref + } + return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path, + Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since, + Position: position, Seq: a.Seq} + } + for _, a := range running { + out.Asks = append(out.Asks, one(a, 0)) + } + for i, a := range waiting { + out.Asks = append(out.Asks, one(a, i+1)) + } + for _, a := range dead { + out.Asks = append(out.Asks, one(a, 0)) + } + return out +} + // queueText is the queue as a person reads it: a summary line, then each kind in queue order. // Never what an ask carries as `held` — that is every artifact the mesh has built. func queueText(q link.Queue, now time.Time) string { @@ -92,7 +170,7 @@ func queueText(q link.Queue, now time.Time) string { return "asked " + now.Sub(t).Round(time.Second).String() + " ago" } what := func(a link.QueuedAsk) string { - s := a.Repository + s := link.WithoutPassword(a.Repository) if a.Path != "" { s += " at " + a.Path } diff --git a/cmd/mesh-controller/queue_answer_test.go b/cmd/mesh-controller/queue_answer_test.go new file mode 100644 index 00000000..d56abddd --- /dev/null +++ b/cmd/mesh-controller/queue_answer_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/link" +) + +// The queue answers as data as well as text (novox/hq issue 455): each ask's repository, commit, kind, +// machine, since and position among the waiting, read from the same queue the table is written from. + +const aCommit = "0123456789abcdef0123456789abcdef01234567" + +func aQueueOfEachKind(now time.Time) link.Queue { + asked := now.Add(-5 * time.Minute) + started := now.Add(-2 * time.Minute) + return link.Queue{Seat: link.TheBuildMachine, Delivered: 2, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{ + {Seq: 1, ID: "build-dead", Repository: "ssh://forge/novox/x", AskedAt: asked, State: link.AskDead}, + {Seq: 2, ID: "build-running", Repository: "ssh://forge/novox/mesh-host", Ref: aCommit, AskedAt: asked, + State: link.AskInFlight, On: "ace", Started: started, + Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}}, + {Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked, + State: link.AskWaiting}, + {Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit, + AskedAt: now.Add(-time.Minute), State: link.AskWaiting, + Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}}, + }} +} + +func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + q := aQueueOfEachKind(now) + got := queueAnswerOf(q, now) + if got.Waiting != 2 || got.InFlight != 1 || got.Dead != 1 || got.Seat != link.TheBuildMachine { + t.Fatalf("counts: %+v", got) + } + if got.Text != queueText(q, now) { + t.Fatalf("the data's text is not the table read from the same queue:\n%s", got.Text) + } + byID := map[string]queuedAnswer{} + for _, a := range got.Asks { + byID[a.ID] = a + } + running := byID["build-running"] + if running.Kind != "check" || running.Machine != "ace" || running.Commit != aCommit || running.Position != 0 || + !running.Since.Equal(now.Add(-2*time.Minute)) || running.State != link.AskInFlight { + t.Errorf("in flight: %+v", running) + } + first, second := byID["build-first"], byID["build-second"] + if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" || + !first.Since.Equal(now.Add(-5*time.Minute)) { + t.Errorf("first waiting: %+v", first) + } + if second.Position != 2 || second.Kind != "check" || second.Commit != aCommit || second.Path != "modules/gitea" || + second.Repository != "ssh://forge/novox/mesh-catalog" { + t.Errorf("second waiting: %+v", second) + } + if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead { + t.Errorf("dead: %+v", dead) + } + // In the table's order: in flight, then the waiting by position, then the dead. + var order []string + for _, a := range got.Asks { + order = append(order, a.ID) + } + if want := []string{"build-running", "build-first", "build-second", "build-dead"}; !reflect.DeepEqual(order, want) { + t.Errorf("order %v, want %v", order, want) + } +} + +// The verb asks its command for JSON, and its answer carries the data while its output stays the table a +// person reads, so a text caller reads what it always did. +func TestTheQueueVerbKeepsTheTableAndCarriesTheData(t *testing.T) { + argv, err := argvFor("queue", nil) + if err != nil || !reflect.DeepEqual(argv, []string{"queue", "--json"}) { + t.Fatalf("queue's command line: %v %v", argv, err) + } + now := time.Now() + q := aQueueOfEachKind(now) + body, err := json.Marshal(queueAnswerOf(q, now)) + if err != nil { + t.Fatal(err) + } + answer := answerOf(argv, body, "", true) + if answer.Output != queueText(q, now) { + t.Fatalf("the output is not the table:\n%s", answer.Output) + } + data, ok := answer.Answer.(map[string]any) + if !ok { + t.Fatalf("no data: %#v", answer.Answer) + } + if _, doubled := data["text"]; doubled { + t.Error("the table is said twice: once as the output and again inside the data") + } + asks, _ := data["asks"].([]any) + if len(asks) != 4 || data["waiting"] != float64(2) { + t.Fatalf("data: %v", data) + } + second := asks[2].(map[string]any) + if second["position"] != float64(2) || second["kind"] != "check" || second["commit"] != aCommit { + t.Errorf("second waiting: %v", second) + } + for _, a := range asks { + if _, held := a.(map[string]any)["held"]; held { + t.Error("an ask carries what the mesh holds") + } + } + // A failure is said as text, as before. + failed := answerOf(argv, nil, "mesh-controller: the bus cannot be reached\n", false) + if failed.Answer != nil || !strings.Contains(failed.Output, "cannot be reached") { + t.Errorf("a failure: %+v", failed) + } +} + +// A repository named with a password never shows it, in the table or the data. +func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) { + if got := link.WithoutPassword("https://agent:s3cret@forge.example/novox/x.git"); strings.Contains(got, "s3cret") || + !strings.Contains(got, "agent@forge.example/novox/x.git") { + t.Errorf("redacted: %s", got) + } + now := time.Now() + q := link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{{Seq: 1, ID: "build-1", + Repository: "https://agent:s3cret@forge.example/novox/x.git", State: link.AskWaiting}}} + body, _ := json.Marshal(queueAnswerOf(q, now)) + if strings.Contains(string(body), "s3cret") { + t.Errorf("the queue said a password: %s", body) + } + for _, kept := range []string{"ssh://git@forge:222/novox/x.git", "novox/x", "git@forge:novox/x.git", ""} { + if got := link.WithoutPassword(kept); got != kept { + t.Errorf("%q became %q", kept, got) + } + } +} diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 5766396a..16096151 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -326,7 +326,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) { args map[string]any want []string }{ - {"queue", nil, []string{"queue"}}, + {"queue", nil, []string{"queue", "--json"}}, {"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}}, {"clear", nil, []string{"clear"}}, {"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}}, diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 2b013154..e84b5d0a 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -426,7 +426,8 @@ func (a *verbArguments) commandLine() ([]string, error) { return argv, nil // The build queue (novox/hq ADR 0219). case "queue": - return []string{"queue"}, nil + // As data, the table its output (novox/hq issue 455). + return []string{"queue", "--json"}, nil case "cancel", "kill": if err := need("id"); err != nil { return nil, err @@ -977,7 +978,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co // What the records say the registries may keep (novox/hq ADR 0251). "artifacts": true, "collect": true, "images": true, "mirrors": true, // The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239. - "delivery": true} + "delivery": true, + // The build queue, each ask with its position among the waiting (novox/hq issue 455). + "queue": true} + +// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the +// verb answers that table as its output, as it did before it answered data, and the rest as `answer` — +// both from the one read, so they never disagree (novox/hq issue 455). +var tableBesideVerbs = map[string]bool{"queue": true} // overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the // command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs @@ -1097,6 +1105,12 @@ func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer { var parsed any if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil { answer.Answer = parsed + if doc, isDoc := parsed.(map[string]any); isDoc && tableBesideVerbs[argv[0]] { + if table, said := doc["text"].(string); said { + answer.Output = table + stderr + delete(doc, "text") + } + } if overviewVerbs[argv[0]] { // Once, as data: the same document again as text doubled an answer that already // outgrew one message of the bus (novox/hq issue 314). diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index bf3d8e2d..c7b4f6fc 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -256,7 +256,7 @@ func stringLit(e ast.Expr) (string, bool) { var accountedFlags = map[string]map[string]string{ "status": {"json": "set by the verb: the answer is data"}, "seats": {"json": "set by the verb: the answer is data"}, - "queue": {"json": "not set: the verb answers the table a person reads"}, + "queue": {"json": "set by the verb: the answer is data, its output the table (novox/hq issue 455)"}, "plan": {"json": "set by the verb unless files is asked"}, "push": { "wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply", diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 013f5c77..e613eb3e 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -333,7 +333,9 @@ var ControllerVerbs = []Verb{ // The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a // failed outcome for it, so a plan waiting on it fails visibly instead of hanging. {Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " + - "and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked.", + "and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked. " + + "Answers the table as text and, as data, each ask's state, kind (check or build), repository, commit, machine, since " + + "and position among the waiting (novox/hq issue 455).", Input: schema(nil, nil)}, {Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " + "cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.", diff --git a/internal/link/queue.go b/internal/link/queue.go index 86c9ab07..41abbb9e 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -7,6 +7,7 @@ import ( "fmt" "regexp" "sort" + "strings" "time" "github.com/nats-io/nats.go" @@ -268,6 +269,30 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e return q, nil } +// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written +// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given, +// which a cancel's outcome and a rebuild's match are read from. +func WithoutPassword(repository string) string { + scheme, rest, found := strings.Cut(repository, "://") + if !found { + return repository + } + authority, path, _ := strings.Cut(rest, "/") + at := strings.LastIndex(authority, "@") + if at < 0 { + return repository + } + user, _, password := strings.Cut(authority[:at], ":") + if !password { + return repository + } + out := scheme + "://" + user + authority[at:] + if len(rest) > len(authority) { + out += "/" + path + } + return out +} + // ReadBuildEvents is every start and outcome the seat announced since a moment, from the events // stream, with a consumer of its own that is gone when this returns. func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) {