novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

push --behind, and a builder told where to publish

Browse Source 
`status` says which machines are not doing what they were told, and
nothing acted on it: a machine that refused or failed stayed wrong until
somebody ran push again naming it.

`push --behind` sends only to machines whose last report was not a clean
apply. A command rather than a timer, deliberately: a scheduler is then a
scheduler over this, where building the scheduler first would have meant
two paths to one act with nothing to compare them against.

Naming a machine and asking which machines need one are different
requests, so `push <node> --behind` is refused rather than guessed. With
nothing behind it says so, because "nothing needed one" and "this did not
run" must never look the same. A machine failing the same way for six
hours is pushed to anyway and said about — refusing would leave no way to
retry after fixing the cause, and this is a command somebody ran.

Proven in the lab: a machine is broken with a package that does not
exist, `push --behind` names it and not the machine that is fine, the
module is corrected, and the machine recovers without anybody naming it.

And the builder can be told where to publish rather than configured. A
builder that is a module requires an artifact store, and the mesh writes
it the same binding any consumer of any provision gets. A binding with no
address is refused rather than falling back to anything — that would
publish to a store on the wrong machine and be found out much later. The
variable remains for a builder run by a person, which is how it is still
run while being developed.
This commit is contained in:
jschoubben
2026-08-30 19:47:02 +02:00
parent 45c3853f4e
commit 79d6ade4c8
3 changed files with 184 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-builder/main.go
+53 -5
View File
@@ -47,7 +47,8 @@ It consumes build requests and answers with what it made. Nothing is listened on
is dialled except the broker.
MESH_BROKER_AMQP where the broker is, with this builder's own credential
MESH_REGISTRY host:port to publish artifacts to
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
MESH_BINDING a file the mesh wrote saying where the artifact store is
MESH_WORKSPACE where to clone and build (default: a temporary directory)
`
@@ -67,10 +68,9 @@ func run() error {
if amqpURL == "" {
return fmt.Errorf("no MESH_BROKER_AMQP: a builder with no broker has nothing to build")
}
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
if registry == "" {
return fmt.Errorf(
"no MESH_REGISTRY: a built artifact nobody can fetch is not built")
registry, err := whereToPublish()
if err != nil {
return err
}
workspace := os.Getenv("MESH_WORKSPACE")
if workspace == "" {
@@ -212,3 +212,51 @@ func short(commit string) string {
}
return commit
}
// whereToPublish is the artifact store this builder uses.
//
// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the
// mesh writes it a file saying which machine answers that and on what port — the same binding any
// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in
// step by hand, and moving the store is an ordinary reassignment rather than an edit on every
// build machine.
//
// The environment variable remains for a builder run by a person, which is how this started and
// how it is still run while being developed.
func whereToPublish() (string, error) {
binding := strings.TrimSpace(os.Getenv("MESH_BINDING"))
if binding == "" {
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
if registry == "" {
return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " +
"nobody can fetch is not built")
}
return registry, nil
}
raw, err := os.ReadFile(binding)
if err != nil {
return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err)
}
var told struct {
From string `json:"from"`
At string `json:"at"`
Serves map[string]any `json:"serves"`
}
if err := json.Unmarshal(raw, &told); err != nil {
return "", fmt.Errorf("%s is not a binding: %w", binding, err)
}
if told.At == "" {
// The provider is not on the private network, so there is no name to reach it by. Said
// rather than falling back to the machine's own name, which would publish to a store on
// the wrong machine and be found out much later.
return "", fmt.Errorf(
"%s says the artifact store is on %q and gives no address for it", binding, told.From)
}
port, ok := told.Serves["port"]
if !ok {
return "", fmt.Errorf("%s says nothing about which port the artifact store answers on",
binding)
}
return fmt.Sprintf("%s:%v", told.At, port), nil
}
cmd/mesh-builder/where_test.go
+76
View File
@@ -0,0 +1,76 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// Where a builder publishes.
//
// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh
// writes it a binding saying which machine answers and on what port. Reading it means the address
// is not a setting somebody keeps in step by hand.
func binding(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "artifact-store.json")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func TestTheMeshSaysWhereToPublish(t *testing.T) {
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store",
"from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`))
where, err := whereToPublish()
if err != nil {
t.Fatal(err)
}
if where != "anchor.internal:5000" {
t.Fatalf("got %q", where)
}
}
func TestABindingWithNoAddressIsRefused(t *testing.T) {
// The provider is not on the private network, so there is no name to reach it by. Falling
// back to anything would publish to a store on the wrong machine and be found out much later.
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`))
_, err := whereToPublish()
if err == nil {
t.Fatal("a binding with nowhere to reach was accepted")
}
if !strings.Contains(err.Error(), "anchor") {
t.Fatalf("the failure does not name the machine: %v", err)
}
}
func TestABindingWithNoPortIsRefused(t *testing.T) {
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`))
if _, err := whereToPublish(); err == nil {
t.Fatal("a binding saying nothing about a port was accepted")
}
}
func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) {
// Which is how this started and how it is still run while being developed.
t.Setenv("MESH_BINDING", "")
t.Setenv("MESH_REGISTRY", "127.0.0.1:5000")
where, err := whereToPublish()
if err != nil {
t.Fatal(err)
}
if where != "127.0.0.1:5000" {
t.Fatalf("got %q", where)
}
}
func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) {
t.Setenv("MESH_BINDING", "")
t.Setenv("MESH_REGISTRY", "")
if _, err := whereToPublish(); err == nil {
t.Fatal("a builder with nowhere to publish reported somewhere")
}
}
cmd/mesh-control/main.go
+55 -2
View File
@@ -145,7 +145,7 @@ func usage() {
pin <node> <provision> <from> which node this one gets a provision from
unpin <node> <provision> put that question back
plan <node> [--files|--json] what that node would run, and why
push [<node>] send a node everything it should be
push [<node>] [--behind] send a node everything it should be, or only those that need it
version what this binary is
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
@@ -1406,8 +1406,28 @@ func planCommand(ctx context.Context, args []string) error {
// half-configured for as long as that lasts — and the two are computed from the same picture of
// the mesh, so sending them apart would let them disagree.
func pushCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("push", flag.ContinueOnError)
// Only the machines that need it.
//
// **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is
// a scheduler over this, and building the scheduler first would mean two paths to one act
// with nothing to compare them against. A person can run this; so can cron; so can whatever
// eventually watches.
behind := set.Bool("behind", false,
"only machines whose last declaration was refused or partly failed")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
args = positionals
if len(args) > 1 {
return errors.New("push [<node>] — one node, or all of them")
return errors.New("push [<node>] [--behind] — one node, or all of them")
}
if len(args) == 1 && *behind {
// Naming a machine and asking for the ones that need it are two different requests, and
// guessing which was meant would sometimes push to a machine somebody did not name.
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
"other asks which machines need one")
}
inv, err := openInventory(ctx)
if err != nil {
@@ -1428,6 +1448,25 @@ func pushCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// Which machines are not in the state they were sent, when that is what was asked for.
var needsOne map[string]inventory.Doing
if *behind {
wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
if err != nil {
return err
}
needsOne = map[string]inventory.Doing{}
for _, d := range wrong {
needsOne[d.Node] = d
}
if len(needsOne) == 0 {
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
// must never look the same.
fmt.Println("every machine is doing what it was told")
return nil
}
}
gens, err := generators(ctx, inv)
if err != nil {
return err
@@ -1453,6 +1492,20 @@ func pushCommand(ctx context.Context, args []string) error {
if len(args) == 1 && n.Name != args[0] {
continue
}
if *behind {
doing, needs := needsOne[n.Name]
if !needs {
continue
}
// A machine that has been failing the same way for a long time is not going to stop
// because it was asked again. Said, and pushed to anyway — refusing would leave no
// way to retry after fixing the cause, and this is a command somebody ran.
if since := time.Since(doing.At); since > 6*time.Hour {
fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+
"unlikely to be timing\n",
n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04"))
}
}
plan, settings, err := planFor(ctx, inv, n.Name)
if err != nil {
refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err))