novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 7 Releases Wiki Activity

Merge pull request 'Three verbs for the registries: artifacts, collect, images (hq ADR 0251)' (#140) from feat/registry-verbs into main

Browse Source
This commit was merged in pull request #140.
This commit is contained in:
mesh-admin
2026-10-08 11:05:35 +00:00
parent 38928c4e1f 557b23d43f
commit a8f60f1f69
12 changed files with 1116 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/catalogue/verbs.go
+28
View File
@@ -435,6 +435,34 @@ var ControllerVerbs = []Verb{
"behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " +
"past the commit it was built from, bases asked first",
}, nil, "behind")},
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51).
{Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " +
"each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " +
"it, or one of the five most recent builds of a module the mesh holds — or eligible: made, kept for no " +
"reason, not yet let go. Collected ones are counted, and listed only with collected. Reads only " +
"(novox/hq ADR 0251).",
Input: schema(map[string]string{
"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
"collected": "\"true\": list the collected references too (they only grow; counted always)",
}, nil, "collected")},
{Name: "collect", Description: "The artifact store's sweep, now: what the mesh made and keeps for no reason " +
"is let go of. Without confirm a dry run — every kept archive asked about with HEADs, what would be let go " +
"listed, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible " +
"artifact let go of, oldest first, and recorded collected — a hand act, recorded with its why. Bounded by " +
"most (500 by default, at most 5000) and 45 seconds; what is left is said. Bytes are reclaimed by the " +
"store's nightly collector. Never touches a digest the mesh did not record (novox/hq ADR 0189, ADR 0251).",
Input: schema(map[string]string{
"confirm": "\"true\": let go of what is eligible (needs why); without it nothing is done",
"why": "with confirm: why — required, and recorded in the hand-act log",
"most": "let go of at most this many (500 by default, at most 5000)",
}, nil, "confirm")},
{Name: "images", Description: "Every container image one machine's declaration names — the declaration the " +
"mesh would send it now and the one it was last sent — each with the resources naming it: what the " +
"machine keeps when it prunes its images. sent_known is false when the last one sent is not kept with " +
"its images. Reads only (novox/hq ADR 0251).",
Input: schema(map[string]string{
"node": "the machine's name",
}, []string{"node"})},
}
// schema is a JSON schema for an object of string properties, which is every argument the verbs
internal/inventory/artifacts_test.go
+66
View File
@@ -0,0 +1,66 @@
package inventory
import (
"context"
"fmt"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Every artifact made has one state, and a kept one says why** (novox/hq ADR 0251): what the
// store's own tools set beside what the store holds, so an operator reads why something stays.
func TestEveryArtifactMadeHasAStateAndAKeptOneSaysWhy(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
var made []string
for i := 1; i <= 8; i++ {
made = append(made, built(t, inv, fmt.Sprintf("b%02d", i), "web", i))
}
// The definition names the oldest and the newest: the newest is kept for both reasons.
m := catalogue.Manifest{Module: "web", Version: "1", Resources: []map[string]any{
{"id": "app", "type": "container", "name": "web", "image": made[0]},
{"id": "next", "type": "container", "name": "web-next", "image": made[7]},
}}
if err := inv.RegisterModule(ctx, m, Source{Repository: "https://forge.invalid/web.git"}); err != nil {
t.Fatal(err)
}
if err := inv.MarkCollected(ctx, []string{made[1]}); err != nil {
t.Fatal(err)
}
states, err := inv.Artifacts(ctx)
if err != nil {
t.Fatal(err)
}
if len(states) != len(made) {
t.Fatalf("%d states for %d artifacts made", len(states), len(made))
}
want := []struct {
state string
why []string
}{
{ArtifactKept, []string{KeptByDefinition}},
{ArtifactCollected, nil},
{ArtifactEligible, nil},
{ArtifactKept, []string{KeptByRecentBuild}},
{ArtifactKept, []string{KeptByRecentBuild}},
{ArtifactKept, []string{KeptByRecentBuild}},
{ArtifactKept, []string{KeptByRecentBuild}},
{ArtifactKept, []string{KeptByRecentBuild, KeptByDefinition}},
}
for i, s := range states {
if s.Reference != made[i] {
t.Fatalf("state %d is of %s, want %s: the oldest first", i, s.Reference, made[i])
}
if s.State != want[i].state || fmt.Sprint(s.Why) != fmt.Sprint(want[i].why) {
t.Errorf("%s: %s %v, want %s %v", s.Reference, s.State, s.Why, want[i].state, want[i].why)
}
}
collect, err := inv.ToCollect(ctx)
if err != nil {
t.Fatal(err)
}
if len(collect) != 1 || collect[0] != made[2] {
t.Errorf("ToCollect is %v, want only the eligible %s", collect, made[2])
}
}
internal/inventory/collection.go
+88 -50
View File
@@ -27,7 +27,39 @@ import (
// release that turns out wrong can be taken.
const KeptBuilds = 5
// ToCollect is every artifact the mesh made, no longer keeps, and has not already collected.
// Why an artifact is kept: the words `artifacts` answers (novox/hq ADR 0251).
const (
// KeptByDefinition: a definition the mesh holds names it, so it could be handed to a machine now.
KeptByDefinition = "definition"
// KeptByRecentBuild: it is an artifact of one of the KeptBuilds most recent successful builds of a
// module the mesh still holds — somewhere a release that turns out wrong can go back to.
KeptByRecentBuild = "recent-build"
// KeptUnaddressable: it names no digest, so nothing here can speak for it, and it is kept rather
// than guessed about.
KeptUnaddressable = "unaddressable"
)
// The states an artifact the mesh recorded making is in (novox/hq ADR 0251).
const (
ArtifactKept = "kept"
ArtifactEligible = "eligible"
ArtifactCollected = "collected"
)
// ArtifactState is one artifact the mesh recorded making, and what the records say of it now.
type ArtifactState struct {
Reference string
State string
// Why is the reasons a kept artifact is kept; empty for any other.
Why []string
}
// Artifacts is every artifact any successful build recorded, oldest first, each with its state:
// kept (and why), eligible to let go, or already collected (novox/hq ADR 0189, ADR 0251).
//
// **One reading of the records for every question asked of them**: the sweep's ToCollect, the
// operator's `artifacts` and the store's own tools read the same states, so what one says may go is
// what the others say is eligible.
//
// Three reasons an artifact stays, and nothing else is a reason:
//
@@ -38,57 +70,62 @@ const KeptBuilds = 5
// nothing beyond what a held definition names (novox/hq issue 253);
// - it was already collected, in which case there is nothing left to do.
//
// Returned in a stated order so two runs over the same records ask for the same things in the
// same sequence, which is what makes a failed sweep safe to simply run again.
func (i *Inventory) ToCollect(ctx context.Context) ([]string, error) {
keep, err := i.keptReferences(ctx)
// In a stated order, so two runs over the same records ask for the same things in the same
// sequence, which is what makes a failed sweep safe to simply run again.
func (i *Inventory) Artifacts(ctx context.Context) ([]ArtifactState, error) {
keep, err := i.KeepSet(ctx)
if err != nil {
return nil, err
}
rows, err := i.store.Pool().Query(ctx,
// Every artifact of every successful build, oldest first, minus what has already been
// collected. A failed build published nothing, so it names nothing to remove.
`select b.made
from build b
where b.failed = '' and b.module is not null and b.module <> ''
order by b.at asc, b.id asc`)
all, err := i.everyReferenceMade(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
collected, err := i.alreadyCollected(ctx)
if err != nil {
return nil, err
}
seen := map[string]bool{}
var out []string
for rows.Next() {
var raw []byte
if err := rows.Scan(&raw); err != nil {
return nil, err
}
var made []Artifact
if err := json.Unmarshal(raw, &made); err != nil {
// One unreadable record must not stop the rest being collected — and an artifact this
// row named is simply not offered, which errs toward keeping.
continue
}
for _, a := range made {
reference := asRecorded(a.Reference)
if reference == "" || keep[reference] || collected[reference] || seen[reference] {
continue
}
seen[reference] = true
out = append(out, reference)
out := make([]ArtifactState, 0, len(all))
for _, reference := range all {
switch {
case len(keep[reference]) > 0:
out = append(out, ArtifactState{Reference: reference, State: ArtifactKept, Why: keep[reference]})
case collected[reference]:
out = append(out, ArtifactState{Reference: reference, State: ArtifactCollected})
default:
out = append(out, ArtifactState{Reference: reference, State: ArtifactEligible})
}
}
return out, rows.Err()
return out, nil
}
// keptReferences is every artifact reference the mesh still keeps, for either of the two reasons.
func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error) {
keep := map[string]bool{}
// ToCollect is every artifact the mesh made, no longer keeps, and has not already collected —
// oldest first (Artifacts says why each other one stays).
func (i *Inventory) ToCollect(ctx context.Context) ([]string, error) {
states, err := i.Artifacts(ctx)
if err != nil {
return nil, err
}
var out []string
for _, s := range states {
if s.State == ArtifactEligible {
out = append(out, s.Reference)
}
}
return out, nil
}
// KeepSet is every artifact reference the mesh still keeps, each with the reasons it is kept.
func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
keep := map[string][]string{}
because := func(reference, why string) {
for _, w := range keep[reference] {
if w == why {
return
}
}
keep[reference] = append(keep[reference], why)
}
// **Whatever a definition the mesh holds names.** Read as text rather than by walking the
// resource shapes: a reference may be a container's image, a bundle's source, or a field some
@@ -140,7 +177,7 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
}
for _, a := range made {
if reference := asRecorded(a.Reference); reference != "" {
keep[reference] = true
because(reference, KeptByRecentBuild)
}
}
}
@@ -148,28 +185,28 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
return nil, err
}
// And anything a manifest mentions. Done after the recent set so the scan runs over the
// candidates rather than over every reference ever recorded: a manifest holds a reference
// composed with the store's address or kept bare, so the search is for the digest within it.
// And anything a manifest mentions — every reference ever made, the recent ones included, so a
// kept artifact carries each of its reasons and the operator reads why, not only that (novox/hq
// ADR 0251). A manifest holds a reference composed with the store's address or kept bare, so the
// search is for the digest within it.
if len(named) > 0 {
all, err := i.everyReferenceMade(ctx)
if err != nil {
return nil, err
}
for _, reference := range all {
if keep[reference] {
continue
}
digest := digestIn(reference)
if digest == "" {
// Not something the store holds by digest; nothing here can speak for it, so it
// is kept rather than guessed about.
keep[reference] = true
if len(keep[reference]) == 0 {
because(reference, KeptUnaddressable)
}
continue
}
for _, text := range named {
if strings.Contains(text, digest) {
keep[reference] = true
because(reference, KeptByDefinition)
break
}
}
@@ -186,7 +223,7 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
// reference that is kept because nothing here can speak for it is not one the store can be asked
// about.
func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
keep, err := i.keptReferences(ctx)
keep, err := i.KeepSet(ctx)
if err != nil {
return nil, err
}
@@ -200,10 +237,11 @@ func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
return out, nil
}
// everyReferenceMade is every artifact reference any successful build recorded.
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select made from build where failed = '' and module is not null and module <> ''`)
`select made from build where failed = '' and module is not null and module <> ''
order by at asc, id asc`)
if err != nil {
return nil, err
}