Merge origin/main into feat/335, and pin mesh-host at #98's new head

internal/link/build.go

@@ -92,6 +92,38 @@ type BuildRequest struct {
 	// facts snapshot, raises the throwaway stores the check needs, runs the repository's own
 	// merge-check.sh and answers its verdict. Nothing is built, published or registered.
 	Check *CheckRequest `json:"check,omitempty"`
+	// Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key
+	// of the commit's tree and, where a check build holds the same key, copies it by digest into the module's
+	// namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a
+	// differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never
+	// takes a check's build. A builder from before it ignores it and builds.
+	Reuse *ReuseAsk `json:"reuse,omitempty"`
+}
+
+// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2).
+//
+// **Only a build a verdict names is taken.** Builds are the builds the build seat said, in the outcomes of its checks
+// of the walk's own pull requests, that it made for their merges: each key with its artifacts' digests. The record
+// in the scratch namespace is not trusted alone — the artifact store takes writes from anything on the network, a
+// check runs a pull request's own script on that network, and a record planted under another merge's key would
+// otherwise be registered. The builder takes a scratch build only when a build here names its key and every digest.
+type ReuseAsk struct {
+	Pulls  []PullRef      `json:"pulls,omitempty"`
+	Builds []CheckedBuild `json:"builds,omitempty"`
+}
+
+// CheckedBuild is one build a check's verdict names: the pull request it checked, its ask, and the build.
+type CheckedBuild struct {
+	Pull  PullRef    `json:"pull"`
+	Check string     `json:"check"`
+	Built CheckBuilt `json:"built"`
+}
+
+// PullRef names a pull request.
+type PullRef struct {
+	Owner  string `json:"owner"`
+	Repo   string `json:"repo"`
+	Number int    `json:"number"`
 }
 
 // CheckRequest is what a merge check needs beyond the repository and its head.
@@ -128,6 +160,34 @@ type CheckRequest struct {
 	// repository's own check is not run for a group: each member's pull request runs its own.
 	Group   string        `json:"group,omitempty"`
 	Members []GroupMember `json:"members,omitempty"`
+	// Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds
+	// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
+	// take. A build seat from before it builds nothing, and the walk builds as before.
+	Builds []CheckBuild `json:"builds,omitempty"`
+}
+
+// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from
+// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would.
+type CheckBuild struct {
+	Module     string `json:"module"`
+	Repository string `json:"repository,omitempty"`
+	Path       string `json:"path,omitempty"`
+	Ref        string `json:"ref,omitempty"`
+	Merged     bool   `json:"merged,omitempty"`
+	Tier       int    `json:"tier"`
+	// Skip says why the check cannot build it as the walk would — a module that reads the repository checked
+	// through a context, which a check cannot hand the merge — and nothing after it is built either.
+	Skip string `json:"skip,omitempty"`
+}
+
+// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key
+// and the artifacts, in the scratch namespace — or why it was not built.
+type CheckBuilt struct {
+	Module   string         `json:"module"`
+	Key      string         `json:"key,omitempty"`
+	Made     []MadeArtifact `json:"made,omitempty"`
+	NotBuilt string         `json:"not-built,omitempty"`
+	Took     string         `json:"took,omitempty"`
 }
 
 // GroupMember is one other head of a delivery group's composed check.
@@ -175,6 +235,8 @@ type CheckOutcome struct {
 	Trunk   string            `json:"trunk,omitempty"`
 	Beside  map[string]string `json:"beside,omitempty"`
 	Members []CheckedMember   `json:"members,omitempty"`
+	// Built are the builds it made for its merge (novox/hq ADR 0350 §1).
+	Built []CheckBuilt `json:"built,omitempty"`
 }
 
 // SourceOnSeat names a repository by the seat whose holder serves it and its path there.
@@ -257,9 +319,21 @@ type BuildResult struct {
 	// build; then only identical artifacts make a rebuild no move.
 	SourceFingerprint string `json:"source-fingerprint,omitempty"`
 
+	// Delivery is the how-manyth time the queue handed this ask out (BuildStart.Delivery): more than 1
+	// says the outcome is from a run started again after an earlier taker left it (novox/hq issue 594).
+	Delivery int `json:"delivery,omitempty"`
+
 	// Failed is why, when it did.
 	Failed string `json:"failed,omitempty"`
 
+	// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build
+	// instead of building, is "reused from <check build>" with its digests, and ReusedFrom which; BuiltWhy, on one
+	// that built, why it took none (§3, §4). Both empty from a builder that predates it.
+	ReuseKey   string      `json:"reuse-key,omitempty"`
+	Reused     string      `json:"reused,omitempty"`
+	ReusedFrom *ReusedFrom `json:"reused-from,omitempty"`
+	BuiltWhy   string      `json:"built-why,omitempty"`
+
 	// Source is the request's, echoed: the seat form of the repository, for whoever registers.
 	Source *SourceOnSeat `json:"source,omitempty"`
 
@@ -296,3 +370,25 @@ type MadeArtifact struct {
 	Kind      string `json:"kind"`
 	Reference string `json:"reference"`
 }
+
+// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk
+// commit the check merged it into, the check's ask, and its key.
+type ReusedFrom struct {
+	Pull  PullRef `json:"pull"`
+	Head  string  `json:"head,omitempty"`
+	Trunk string  `json:"trunk,omitempty"`
+	Check string  `json:"check,omitempty"`
+	Key   string  `json:"key"`
+}
+
+// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from <check build>", or "built:
+// <why>". A build seat from before it says neither, and that is said.
+func (r BuildResult) MadeWords() string {
+	switch {
+	case r.Reused != "":
+		return r.Reused
+	case r.BuiltWhy != "":
+		return "built: " + r.BuiltWhy
+	}
+	return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)"
+}

internal/link/builds.go

@@ -92,6 +92,31 @@ type BuildStart struct {
 	Ref        string `json:"ref,omitempty"`
 	On         string `json:"on"`
 	At         string `json:"at"`
+	// Delivery is the how-manyth time the queue handed this ask out: 1 for a first start, more for one
+	// an earlier taker left unfinished — most often a build agent replaced mid-build — and started
+	// again from the beginning (novox/hq issue 594). 0 from a machine that predates it.
+	Delivery int `json:"delivery,omitempty"`
+}
+
+// Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's
+// report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused
+// holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all
+// count as a hand-out. Empty for a first start.
+func Restarted(delivery int) string {
+	if delivery < 2 {
+		return ""
+	}
+	return fmt.Sprintf("restarted: handed out %d times; an earlier hand-out did not finish here, so this run "+
+		"started from the beginning", delivery)
+}
+
+// RunMarker is the short form of Restarted, for a line clipped to a few words — a check's status on the
+// forge (novox/hq issue 594): "(run 2)". Empty for a first start.
+func RunMarker(delivery int) string {
+	if delivery < 2 {
+		return ""
+	}
+	return fmt.Sprintf("(run %d)", delivery)
 }
 
 // BuildLine is one thing a build said while building.

internal/link/builds_nats.go

@@ -315,7 +315,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			// built. A set that cannot be read is said and the ask built: a cancel is a person's
 			// exception, and a holder that refused every build while its grant was missing would
 			// stop the mesh building for it.
-			build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
+			build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
 			if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
 				fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
 			} else if cancelled {
@@ -347,6 +347,18 @@ type natsBuild struct {
 	// seat is the role this build was taken from; what the machine says about it is that role's.
 	seat string
 	seq  int
+	// delivery is the how-manyth time the queue handed this ask out (novox/hq issue 594); 0 when the
+	// server did not say.
+	delivery int
+}
+
+// deliveryOf is how many times the queue has handed this message out, this time included.
+func deliveryOf(msg *nats.Msg) int {
+	meta, err := msg.Metadata()
+	if err != nil {
+		return 0
+	}
+	return int(meta.NumDelivered)
 }
 
 func (b *natsBuild) Request() BuildRequest { return b.request }
@@ -367,6 +379,11 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error {
 	if result.Failed == "" && result.Commit != "" {
 		result.Module = ModuleOf(result.Manifest)
 	}
+	// **Which run this is** (novox/hq issue 594): stamped by the transport, which alone knows, so every
+	// outcome carries it whatever the machine's code filled in.
+	if result.Delivery == 0 {
+		result.Delivery = b.delivery
+	}
 	body, err := json.Marshal(result)
 	if err != nil {
 		return err
@@ -386,11 +403,16 @@ func (b *natsBuild) Done() error { return b.msg.Ack() }
 func (b *natsBuild) Began(ctx context.Context) error {
 	body, err := json.Marshal(BuildStart{
 		ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path,
-		Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano),
+		Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery,
 	})
 	if err != nil {
 		return err
 	}
+	// **A start that is a restart says so in the build's log** (novox/hq issue 594): a reader following
+	// the build sees why it began again, never a silent second run.
+	if said := Restarted(b.delivery); said != "" {
+		b.Say("restarted", said)
+	}
 	publish, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
 	if _, err := b.js.Context().Publish(BuildStartedOf(b.seat), body, nats.Context(publish)); err != nil {

internal/link/builds_nats_test.go

@@ -401,3 +401,97 @@ func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
 		t.Fatal("a machine that could not bind said it takes work")
 	}
 }
+
+// **An ask handed out again says so** (novox/hq issue 594). A build agent replaced mid-build leaves its
+// ask unsettled, and the queue hands it out again to start over from the beginning. That second start
+// must not look like a first one: the machine that takes it says it restarted — in its started event,
+// in the build's log and in its outcome — so the queue and the pull request can say it too.
+func TestNatsAnAskHandedOutAgainSaysItRestarted(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+
+	started, err := js.Conn().SubscribeSync(BuildStarted())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = started.Unsubscribe() }()
+	lines, err := js.Conn().SubscribeSync(BuildLog("b-594"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = lines.Unsubscribe() }()
+	outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = outcomes.Unsubscribe() }()
+	_ = js.Conn().Flush()
+
+	body, _ := json.Marshal(BuildRequest{ID: "b-594", Repository: "/r"})
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+
+	machine := MachineOverNATS(js, "anchor")
+	defer machine.Close()
+	var first sync.Once
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			_ = work.Began(ctx)
+			handedBack := false
+			// The first time it is left unfinished — an agent replaced mid-build — and comes back.
+			first.Do(func() { _ = work.Hold(100 * time.Millisecond); handedBack = true })
+			if handedBack {
+				return
+			}
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"})
+			_ = work.Done()
+		})
+	}()
+
+	var starts []BuildStart
+	for len(starts) < 2 {
+		msg, err := started.NextMsg(10 * time.Second)
+		if err != nil {
+			t.Fatalf("heard %d start(s) of an ask handed out twice: %v", len(starts), err)
+		}
+		var s BuildStart
+		if err := json.Unmarshal(msg.Data, &s); err != nil {
+			t.Fatal(err)
+		}
+		starts = append(starts, s)
+	}
+	if starts[0].Delivery != 1 || starts[1].Delivery != 2 {
+		t.Fatalf("the starts say deliveries %d and %d, want 1 and 2: a restart reads as a first start",
+			starts[0].Delivery, starts[1].Delivery)
+	}
+
+	msg, err := outcomes.NextMsg(10 * time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var result BuildResult
+	if err := json.Unmarshal(msg.Data, &result); err != nil {
+		t.Fatal(err)
+	}
+	if result.Delivery != 2 {
+		t.Fatalf("the outcome says delivery %d, want 2: the pull request cannot say the check restarted",
+			result.Delivery)
+	}
+
+	sawRestarted := false
+	for {
+		msg, err := lines.NextMsg(time.Second)
+		if err != nil {
+			break
+		}
+		var line BuildLine
+		if json.Unmarshal(msg.Data, &line) == nil && line.Step == "restarted" {
+			sawRestarted = true
+		}
+	}
+	if !sawRestarted {
+		t.Fatal("the build's log never said it restarted")
+	}
+}

internal/link/check_builds_test.go

@@ -0,0 +1,39 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+// The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only
+// checks of the walk's pull requests, only builds made, newest first.
+func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) {
+	js := aBus(t)
+	since := time.Now().Add(-time.Minute)
+	say := func(r BuildResult) {
+		body, _ := json.Marshal(r)
+		if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
+	built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
+	say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built,
+		{Module: "web", NotBuilt: "it has no reuse key"}}}})
+	say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8},
+		Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}})
+	say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check
+	newer := built
+	newer.Key = "r1:2"
+	say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}})
+
+	got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" ||
+		got[1].Pull.Number != 7 {
+		t.Fatalf("read %+v", got)
+	}
+}

internal/link/events.go

@@ -290,6 +290,9 @@ type Checked struct {
 	Trunk  string            `json:"trunk,omitempty"`
 	Beside map[string]string `json:"beside,omitempty"`
 	Took   string            `json:"took,omitempty"`
+	// Built are the builds the check made for its merge, each with its reuse key and digests, or why not
+	// (novox/hq ADR 0350 §1): what the walk of the merge takes instead of building.
+	Built []CheckBuilt `json:"built,omitempty"`
 }
 
 // CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq

internal/link/protocol.go

@@ -608,7 +608,10 @@ type Wait struct {
 	Part    string `json:"part"`
 	Secret  string `json:"secret,omitempty"`
 	Setting string `json:"setting,omitempty"`
-	What    string `json:"what"`
+	// Provision is a provision the module requires whose credential only a person can give and nobody gave yet
+	// (novox/hq ADR 0337 §5): a wait names exactly one of Secret, Setting or Provision.
+	Provision string `json:"provision,omitempty"`
+	What      string `json:"what"`
 }
 
 // ResourceHealth is one long-running resource's state.

internal/link/queue.go

@@ -73,6 +73,9 @@ type QueuedAsk struct {
 	On      string    `json:"on,omitempty"`
 	Was     string    `json:"was-on,omitempty"`
 	Started time.Time `json:"started,omitempty"`
+	// Delivery is the how-manyth time the ask was handed out, as its latest start said: more than 1 is
+	// a build started again after an earlier taker left it (novox/hq issue 594).
+	Delivery int `json:"delivery,omitempty"`
 }
 
 // Queue is the seat's work queue as it stands.
@@ -162,7 +165,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
 		}
 		a.State = AskDead
 		if s, ok := current[a.ID]; ok {
-			a.On, a.Started = s.On, startedAt(s)
+			a.On, a.Started, a.Delivery = s.On, startedAt(s), s.Delivery
 			running = append(running, i)
 			continue
 		}
@@ -183,7 +186,7 @@ func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard Bui
 		for _, i := range group {
 			if slots == 0 {
 				// Past what the worker counts: handed out as often as it may be.
-				out[i].On, out[i].Started = "", time.Time{}
+				out[i].On, out[i].Started, out[i].Delivery = "", time.Time{}, 0
 				continue
 			}
 			out[i].State = AskInFlight
@@ -361,6 +364,64 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
 	return heard, nil
 }
 
+// ReadCheckBuilds is every build the seat's checks of those pull requests said they made for their merges since a
+// moment (novox/hq ADR 0350 §2): read from the seat's own outcomes on the events stream, which only its holders may
+// publish, never from the artifact store. Newest first.
+func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pulls []PullRef, since time.Time) ([]CheckedBuild, error) {
+	if len(pulls) == 0 {
+		return nil, nil
+	}
+	wanted := map[PullRef]bool{}
+	for _, p := range pulls {
+		wanted[p] = true
+	}
+	subject := BuildOutcomeOf(seat)
+	sub, err := js.Context().PullSubscribe(subject, "",
+		nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone())
+	if err != nil {
+		return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	info, err := sub.ConsumerInfo()
+	if err != nil {
+		return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
+	}
+	var out []CheckedBuild
+	for remaining := info.NumPending; remaining > 0; {
+		batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
+		if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
+			return nil, fmt.Errorf("reading %s: %w", subject, err)
+		}
+		if len(batch) == 0 {
+			break
+		}
+		remaining -= uint64(len(batch))
+		for _, msg := range batch {
+			var r BuildResult
+			if json.Unmarshal(msg.Data, &r) != nil || r.Checked == nil || r.Check == nil || r.Failed != "" {
+				continue
+			}
+			pull := PullRef{Owner: r.Checked.Owner, Repo: r.Checked.Repo, Number: r.Checked.Number}
+			if !wanted[pull] {
+				continue
+			}
+			for _, b := range r.Check.Built {
+				if b.NotBuilt == "" && b.Key != "" {
+					out = append(out, CheckedBuild{Pull: pull, Check: r.ID, Built: b})
+				}
+			}
+		}
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+	}
+	// Newest first: the stream's order is the order they were said.
+	for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
+		out[i], out[j] = out[j], out[i]
+	}
+	return out, nil
+}
+
 // --- the cancelled set ----------------------------------------------------------------------
 
 // safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.