The mesh knows where a module came from, and whether it is behind

Delivery is a comparison, not a pipeline: the control plane holds what source
exists and what has been built from it, and the difference is the work. Both
halves are written down now, so "is this current" is a question about two
columns rather than something you find out by building.

`status` answers "did my change go out?", which ADR 0010 names as the real risk
of replacing a pipeline with a comparison -- it is answerable today by opening
a pipeline, and something had to replace that.

  zsh    holds 4f2a9c1e, source has 9e3b7d2a
         running on laptop

The machines are the point. A module being out of date is a fact about the
catalogue; which machines are running last week's version is the thing with
consequences.

Three things this had to get right.

A module with no source is never behind -- it was handed over directly, which
is how a one-off arrives, and saying "out of date" about it would be inventing
a comparison against nothing.

A source nobody has checked is not behind either. Reporting it as behind would
put every module on the list the moment provenance was recorded, which makes
the list say nothing. Fault injection found this: my first test passed with the
guard removed, because both halves were empty strings and compared equal. The
case that actually needed it -- a known commit and an unknown head -- was
untested.

And handing over a manifest by hand does not erase where the module normally
comes from. Fixing something in a hurry is legitimate; silently forgetting its
origin is not, because that record is the only thing that would say afterwards
that a machine is running something nobody can rebuild.

Also fixed the flag parsing, which stopped at the first positional argument and
silently ignored every flag after it -- so `module add thing.json --source x`
recorded no source at all and said it had succeeded. The host's own parser
documents this exact footgun and I wrote it again anyway.

cmd/mesh-control/main.go

@@ -87,6 +87,8 @@ func run() error {
 		return planCommand(ctx, args[1:])
 	case "push":
 		return pushCommand(ctx, args[1:])
+	case "status":
+		return statusCommand(ctx)
 	case "version":
 		fmt.Println(version)
 		return nil
@@ -115,7 +117,9 @@ func usage() {
   overlay show                   the private network, as the mesh computes it
   module add <file>              register a module from its manifest
   module list                    what modules this mesh knows about
+  module moved <name> <commit>   the source has a newer commit than the mesh built
   module forget <name>           remove one, unless a node is running it
+  status                         what the mesh is behind on, and which nodes
   assign <node> <module>         put a module on a node
   unassign <node> <module>       take it off
   plan <node>                    what that node would run, and why
@@ -663,10 +667,18 @@ func moduleCommand(ctx context.Context, args []string) error {
 
 	switch args[0] {
 	case "add":
-		if len(args) != 2 {
-			return errors.New("module add <manifest.json>")
+		set := flag.NewFlagSet("module add", flag.ContinueOnError)
+		repo := set.String("source", "", "where this module comes from")
+		ref := set.String("ref", "", "the branch followed there")
+		commit := set.String("commit", "", "the commit this manifest was read at")
+		positionals, err := parseAround(set, args[1:])
+		if err != nil {
+			return err
 		}
-		raw, err := os.ReadFile(args[1])
+		if len(positionals) != 1 {
+			return errors.New("module add <manifest.json> [--source <repo> --ref <branch> --commit <sha>]")
+		}
+		raw, err := os.ReadFile(positionals[0])
 		if err != nil {
 			return err
 		}
@@ -674,10 +686,23 @@ func moduleCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		if err := inv.RegisterModule(ctx, m); err != nil {
+		// Provenance together or not at all. A source with no commit cannot be compared against
+		// anything, so it would record where the module came from and still never be able to say
+		// the mesh is behind it — which is the one thing recording it is for.
+		if (*repo == "") != (*commit == "") {
+			return errors.New("--source and --commit go together: a source with no commit " +
+				"cannot be compared against anything, and a commit with no source has nothing " +
+				"to be compared with")
+		}
+		if err := inv.RegisterModule(ctx, m, inventory.Source{
+			Repository: *repo, Ref: *ref, BuiltFrom: *commit,
+		}); err != nil {
 			return err
 		}
 		fmt.Printf("%s registered", m.Module)
+		if *commit != "" {
+			fmt.Printf(" from %s", short(*commit))
+		}
 		if len(m.Provides) > 0 {
 			fmt.Printf(", providing %s", strings.Join(m.Provides, ", "))
 		}
@@ -714,6 +739,26 @@ func moduleCommand(ctx context.Context, args []string) error {
 		}
 		return nil
 
+	case "moved":
+		if len(args) != 3 {
+			return errors.New("module moved <name> <commit> — the source has a newer commit")
+		}
+		if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil {
+			return err
+		}
+		from, err := inv.SourceOf(ctx, args[1])
+		if err != nil {
+			return err
+		}
+		if from.Current() {
+			fmt.Printf("%s is current at %s\n", args[1], short(from.Head))
+			return nil
+		}
+		fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
+			args[1], short(from.BuiltFrom), short(from.Head))
+		fmt.Println("  build it and `module add` the result to catch up")
+		return nil
+
 	case "forget":
 		if len(args) != 2 {
 			return errors.New("module forget <name>")
@@ -725,7 +770,7 @@ func moduleCommand(ctx context.Context, args []string) error {
 		return nil
 
 	default:
-		return fmt.Errorf("module has no %q; it has add, list and forget", args[0])
+		return fmt.Errorf("module has no %q; it has add, list, moved and forget", args[0])
 	}
 }
 
@@ -934,3 +979,81 @@ func pushCommand(ctx context.Context, args []string) error {
 	fmt.Printf("\n%d node(s) told\n", len(sending))
 	return nil
 }
+
+// short is a commit as a person refers to it.
+func short(commit string) string {
+	if len(commit) > 8 {
+		return commit[:8]
+	}
+	return commit
+}
+
+// statusCommand answers "did my change go out?".
+//
+// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
+// comparison: it is answerable today by opening a pipeline, and something has to replace that or
+// this is worse to live with whatever its other properties.
+//
+// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
+// source now has, and which machines are running the old one.
+func statusCommand(ctx context.Context) error {
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return err
+	}
+	defer inv.Close()
+
+	behind, err := inv.Behind(ctx)
+	if err != nil {
+		return err
+	}
+	if len(behind) == 0 {
+		fmt.Println("every module with a source is built from what that source has")
+		return nil
+	}
+
+	var names []string
+	for m := range behind {
+		names = append(names, m)
+	}
+	sort.Strings(names)
+
+	fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
+	for _, m := range names {
+		from, err := inv.SourceOf(ctx, m)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("  %-20s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
+		if nodes := behind[m]; len(nodes) > 0 {
+			// The part somebody actually wants. A module being out of date is a fact about the
+			// catalogue; machines running the old one is the thing with consequences.
+			fmt.Printf("  %-20s running on %s\n", "", strings.Join(nodes, ", "))
+		} else {
+			fmt.Printf("  %-20s assigned to nothing\n", "")
+		}
+	}
+	return nil
+}
+
+// parseAround reads flags that may sit before, after or between positional arguments.
+//
+// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
+// parses no flags at all and silently ignores every one of them. The host learned this the same
+// way and says so in its own parser: a flag that is quietly dropped is the fault this project
+// keeps naming, and it looks exactly like success.
+func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
+	var positionals []string
+	rest := args
+	for {
+		if err := set.Parse(rest); err != nil {
+			return nil, err
+		}
+		rest = set.Args()
+		if len(rest) == 0 {
+			return positionals, nil
+		}
+		positionals = append(positionals, rest[0])
+		rest = rest[1:]
+	}
+}

internal/inventory/catalogue.go

@@ -20,25 +20,135 @@ var ErrNoSuchModule = errors.New("no module of that name")
 // removing the record would leave the mesh unable to describe what is on it.
 var ErrStillAssigned = errors.New("that module is still assigned to nodes")
 
+// Source is where a module comes from and what has been built from it.
+type Source struct {
+	Repository string
+	Ref        string
+	// BuiltFrom is the commit the manifest the mesh holds was read at.
+	BuiltFrom string
+	// Head is the newest commit the source is known to have.
+	Head string
+}
+
+// Current reports whether what the mesh holds is what the source last had.
+//
+// A module with no source is always current: it was handed over directly, and there is nothing
+// it could be behind. Saying "out of date" about it would be inventing a comparison.
+func (s Source) Current() bool {
+	if s.Repository == "" || s.Head == "" {
+		return true
+	}
+	return s.BuiltFrom == s.Head
+}
+
 // RegisterModule records a module, replacing what was there.
 //
 // Replacing rather than refusing, because a manifest changing is the ordinary case -- a module
 // gains a requirement, a claim, a resource. What matters is that the change is visible the next
 // time a node is resolved, which it is.
-func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest) error {
+func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, from Source) error {
 	raw, err := json.Marshal(m)
 	if err != nil {
 		return err
 	}
+
+	// A module registered without provenance keeps whatever it had. Handing over a manifest by
+	// hand is a legitimate way to fix something in a hurry, and it should not silently erase the
+	// record of where the module normally comes from — which is the only thing that would say,
+	// afterwards, that the machine is running something nobody can rebuild.
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into module (name, manifest, version) values ($1, $2, nullif($3,''))
-		 on conflict (name) do update set manifest = excluded.manifest,
+		`insert into module (name, manifest, version, source, ref, built_from, source_head)
+		 values ($1, $2, nullif($3,''), nullif($4,''), nullif($5,''), nullif($6,''), nullif($6,''))
+		 on conflict (name) do update set
+		     manifest   = excluded.manifest,
 		     version    = excluded.version,
-		                                  registered = now()`,
-		m.Module, raw, m.Version)
+		     registered = now(),
+		     source     = coalesce(excluded.source, module.source),
+		     ref        = coalesce(excluded.ref, module.ref),
+		     built_from = coalesce(excluded.built_from, module.built_from),
+		     source_head = coalesce(excluded.built_from, module.source_head)`,
+		m.Module, raw, m.Version, from.Repository, from.Ref, from.BuiltFrom)
 	return err
 }
 
+// SourceMoved records that a module's source has a newer commit than the mesh has built.
+//
+// This is the whole of noticing. Nothing here builds anything — it writes down that the two
+// halves differ, which is what makes *is this current?* answerable without building, and what
+// makes a module that nobody rebuilt visible rather than silent.
+func (i *Inventory) SourceMoved(ctx context.Context, module, head string) error {
+	tag, err := i.store.Pool().Exec(ctx,
+		`update module set source_head = $2, source_seen = now() where name = $1`, module, head)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return fmt.Errorf("%w: %s", ErrNoSuchModule, module)
+	}
+	return nil
+}
+
+// SourceOf is where a module came from and whether the mesh is behind it.
+func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error) {
+	var s Source
+	var repo, ref, built, head *string
+	err := i.store.Pool().QueryRow(ctx,
+		`select source, ref, built_from, source_head from module where name = $1`,
+		module).Scan(&repo, &ref, &built, &head)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return Source{}, fmt.Errorf("%w: %s", ErrNoSuchModule, module)
+	}
+	if err != nil {
+		return Source{}, err
+	}
+	for _, pair := range []struct {
+		from *string
+		to   *string
+	}{{repo, &s.Repository}, {ref, &s.Ref}, {built, &s.BuiltFrom}, {head, &s.Head}} {
+		if pair.from != nil {
+			*pair.to = *pair.from
+		}
+	}
+	return s, nil
+}
+
+// Behind is every module the mesh has not built from what its source now has, with the nodes
+// running the old one.
+//
+// The nodes are the point. "Is this module out of date" is a fact about the catalogue; "which
+// machines are running last week's version" is the question somebody actually has, and it is the
+// one novox/hq ADR 0010 names as the thing that must not be lost.
+func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select m.name, coalesce(n.name, '')
+		   from module m
+		   left join assignment a on a.module = m.name
+		   left join node n on n.id = a.node
+		  where m.source is not null
+		    and m.source_head is not null
+		    and coalesce(m.built_from, '') is distinct from m.source_head
+		  order by m.name, n.name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := map[string][]string{}
+	for rows.Next() {
+		var module, node string
+		if err := rows.Scan(&module, &node); err != nil {
+			return nil, err
+		}
+		if _, seen := out[module]; !seen {
+			out[module] = nil
+		}
+		if node != "" {
+			out[module] = append(out[module], node)
+		}
+	}
+	return out, rows.Err()
+}
+
 // Catalogue is every module the mesh knows about, which is what resolution needs: the question
 // "how many modules provide this" cannot be asked of a subset.
 func (i *Inventory) Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error) {

internal/inventory/catalogue_test.go

@@ -22,7 +22,7 @@ func TestAModuleRoundTripsWholeAndUnshredded(t *testing.T) {
 		Claims:       []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}},
 		Resources:    []map[string]any{{"id": "conf", "type": "file", "path": "/etc/X11/x.conf"}},
 	}
-	if err := inv.RegisterModule(t.Context(), m); err != nil {
+	if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -46,10 +46,10 @@ func TestRegisteringAgainReplacesTheManifest(t *testing.T) {
 	// A manifest changing is the ordinary case — a module gains a requirement, a claim, a
 	// resource. What matters is that the change is what the next resolution sees.
 	inv := fresh(t)
-	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil)); err != nil {
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RegisterModule(t.Context(), manifest("thing", []string{"a-thing"}, nil)); err != nil {
+	if err := inv.RegisterModule(t.Context(), manifest("thing", []string{"a-thing"}, nil), Source{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -72,7 +72,7 @@ func TestAModuleAMachineIsRunningCannotBeForgotten(t *testing.T) {
 	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil)); err != nil {
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
 		t.Fatal(err)
 	}
 	if err := inv.Assign(t.Context(), "laptop", "thing"); err != nil {
@@ -99,7 +99,7 @@ func TestRemovingANodeTakesItsAssignments(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil)); err != nil {
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
 		t.Fatal(err)
 	}
 	if err := inv.Assign(t.Context(), "laptop", "thing"); err != nil {
@@ -146,7 +146,7 @@ func TestAssigningTwiceIsNotAnError(t *testing.T) {
 	if _, err := inv.AddNode(t.Context(), "laptop"); err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil)); err != nil {
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
 		t.Fatal(err)
 	}
 	for i := 0; i < 3; i++ {
@@ -210,3 +210,157 @@ func TestOnlyPresentCapabilitiesCount(t *testing.T) {
 		t.Error("a capability the node reported as ABSENT was counted as present")
 	}
 }
+
+func TestAModuleWithNoSourceIsNeverBehind(t *testing.T) {
+	// It was handed over directly, which is how a one-off arrives and how every module got here
+	// before provenance existed. Saying "out of date" about it would be inventing a comparison
+	// against nothing.
+	inv := fresh(t)
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil {
+		t.Fatal(err)
+	}
+	from, err := inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !from.Current() {
+		t.Error("a module with no source was reported as behind")
+	}
+	behind, err := inv.Behind(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(behind) != 0 {
+		t.Errorf("a module with no source is in the behind list: %v", behind)
+	}
+}
+
+func TestASourceThatMovedMakesTheModuleBehind(t *testing.T) {
+	inv := fresh(t)
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil),
+		Source{Repository: "novox/thing", Ref: "main", BuiltFrom: "aaaa1111"}); err != nil {
+		t.Fatal(err)
+	}
+	from, err := inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !from.Current() {
+		t.Fatal("a module built from the only commit its source has is behind")
+	}
+
+	if err := inv.SourceMoved(t.Context(), "thing", "bbbb2222"); err != nil {
+		t.Fatal(err)
+	}
+	from, err = inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if from.Current() {
+		t.Error("the source moved and the module still reports as current")
+	}
+}
+
+func TestBeingBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
+	// The question somebody actually has. A module being out of date is a fact about the
+	// catalogue; machines running last week's version is the thing with consequences.
+	inv := fresh(t)
+	for _, n := range []string{"laptop", "workstation"} {
+		if _, err := inv.AddNode(t.Context(), n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil),
+		Source{Repository: "novox/thing", BuiltFrom: "aaaa1111"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, n := range []string{"laptop", "workstation"} {
+		if err := inv.Assign(t.Context(), n, "thing"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.SourceMoved(t.Context(), "thing", "bbbb2222"); err != nil {
+		t.Fatal(err)
+	}
+
+	behind, err := inv.Behind(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(behind["thing"]) != 2 {
+		t.Errorf("running on %v; both machines have the old one", behind["thing"])
+	}
+}
+
+func TestRebuildingCatchesUp(t *testing.T) {
+	inv := fresh(t)
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil),
+		Source{Repository: "novox/thing", BuiltFrom: "aaaa1111"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.SourceMoved(t.Context(), "thing", "bbbb2222"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(t.Context(), manifest("thing", []string{"a-thing"}, nil),
+		Source{Repository: "novox/thing", BuiltFrom: "bbbb2222"}); err != nil {
+		t.Fatal(err)
+	}
+
+	from, err := inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !from.Current() {
+		t.Errorf("built from the commit the source has and still behind: %+v", from)
+	}
+}
+
+func TestHandingOverAManifestDoesNotEraseWhereItComesFrom(t *testing.T) {
+	// Fixing something in a hurry is legitimate. Silently forgetting where the module normally
+	// comes from is not: it is the only thing that would say, afterwards, that a machine is
+	// running something nobody can rebuild.
+	inv := fresh(t)
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil),
+		Source{Repository: "novox/thing", Ref: "main", BuiltFrom: "aaaa1111"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(t.Context(), manifest("thing", []string{"patched"}, nil),
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	from, err := inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if from.Repository != "novox/thing" {
+		t.Errorf("handing over a manifest erased the source: %+v", from)
+	}
+}
+
+func TestASourceNobodyHasCheckedIsNotBehind(t *testing.T) {
+	// A module built from a commit, where nothing has yet told the mesh whether that source has
+	// moved. It is not behind — nobody has looked. Reporting it as behind would put every module
+	// on the list the moment provenance was recorded, which makes the list say nothing.
+	inv := fresh(t)
+	if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil),
+		Source{Repository: "novox/thing", BuiltFrom: "aaaa1111"}); err != nil {
+		t.Fatal(err)
+	}
+
+	from, err := inv.SourceOf(t.Context(), "thing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Registering sets the head to what was built, so the two agree until something says
+	// otherwise. Either way it must not read as behind.
+	if !from.Current() {
+		t.Errorf("a source nobody has checked reports as behind: %+v", from)
+	}
+
+	// And with the head genuinely unknown, which is what a module registered before provenance
+	// existed looks like after somebody adds a source to it.
+	if (Source{Repository: "novox/thing", BuiltFrom: "aaaa1111"}).Current() == false {
+		t.Error("a module with no known head reports as behind")
+	}
+}

internal/inventory/migrations/0006-where-a-module-came-from.sql

@@ -0,0 +1,20 @@
+-- Where each module came from, and whether what the mesh holds is still current.
+--
+-- novox/hq ADR 0010: delivery is a comparison, not a pipeline. The control plane holds what
+-- source exists and what has been built from it, and builds the difference. So both halves have
+-- to be written down, and *is this current?* is a question about two columns rather than
+-- something you find out by building.
+
+alter table module add column source      text;  -- where it comes from
+alter table module add column ref         text;  -- the branch followed there
+alter table module add column built_from  text;  -- the commit this manifest was read at
+alter table module add column source_head text;  -- the newest commit the source is known to have
+
+-- When the mesh last learned the source had moved. Kept apart from `registered`, which is when
+-- the manifest last changed: a source that moved and was never built is exactly the state this
+-- exists to make visible, and one timestamp could not show it.
+alter table module add column source_seen timestamptz;
+
+-- A module with no source is not a fault. It was handed over directly -- which is how every
+-- module got here before this existed, and how a one-off still arrives. It is simply never out
+-- of date, because there is nothing it could be behind.