mesh-controller/cmd/mesh-controller/chain.go

package main

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/novox/mesh-controller/internal/chain"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.

// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
func chainHolds(ctx context.Context) error {
	_, err := theLease.epoch(ctx)
	return err
}

// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command.
func openTheChain(ctx context.Context) (*chain.Chain, error) {
	js, err := aBus()
	if err != nil {
		return nil, err
	}
	// Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command.
	return chain.OnTheBus(ctx, js.Conn(), chainHolds)
}

// attachTheChain gives the inventory of the serving controller the chain on its connection.
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
	bus, ok := server.Bus().(link.OverNATS)
	if !ok {
		return errors.New("the chain lives on the bus, and this control plane is not on it")
	}
	c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds)
	if err != nil {
		return err
	}
	inv.UseChain(c)
	inv.RequireLayout()
	// And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so
	// nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a
	// read of the start was refused, and the controller exited and was started again every five seconds. The layout of
	// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
	if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
		fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
		go layBuildsOutSoon(ctx, inv)
	}
	// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
	go askAgainSoon(ctx, inv)
	// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
	go copySentBuildsSoon(ctx, inv)
	return nil
}

// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the
// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a
// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood.
func askAgain(ctx context.Context, inv *inventory.Inventory) {
	pending, err := inv.RequestsToAskAgain(ctx)
	if err != nil {
		fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err)
		return
	}
	for _, r := range pending {
		if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed {
			if err != nil {
				fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err)
			}
			continue
		}
		id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref)
		if err != nil {
			if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil {
				fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr)
			}
			fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err)
			continue
		}
		recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path,
			Ref: r.Ref, Commit: r.Commit, For: r.For})
		if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil {
			fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err)
		}
		fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id)
	}
}

// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again.
func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
	select {
	case <-ctx.Done():
	case <-time.After(time.Minute):
		askAgain(ctx, inv)
	}
}

// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
const kindSentBuildsNotCopied = "sent-builds-not-copied"

// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
// that names a commit and empties each column it has copied.
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
	failed := false
	for {
		_, err := inv.CopySentBuilds(ctx)
		o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
			Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
			Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
		if err != nil {
			fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
			if keeper := conditionsFrom; keeper != nil && !failed {
				o.Summary += err.Error()
				if _, oerr := keeper.Observe(ctx, o); oerr == nil {
					failed = true
				}
			}
		} else {
			if keeper := conditionsFrom; keeper != nil && failed {
				_, _ = keeper.Clear(ctx, o.Key(), "copied")
			}
			return
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(chainMoveRetry):
		}
	}
}

// chainMoveRetry is how long the copy waits before it tries again.
const chainMoveRetry = 30 * time.Second

// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
// has, no build is read, and a send that needs one is refused with the reason.
const kindBuildsNotLaidOut = "builds-not-laid-out"

// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
	failed := false
	for {
		err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
		o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
			Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
			Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
		if err != nil {
			fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
			if keeper := conditionsFrom; keeper != nil && !failed {
				o.Summary += err.Error()
				if _, oerr := keeper.Observe(ctx, o); oerr == nil {
					failed = true
				}
			}
		} else {
			if keeper := conditionsFrom; keeper != nil && failed {
				_, _ = keeper.Clear(ctx, o.Key(), "laid out")
			}
			return
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(chainMoveRetry):
		}
	}
}