mesh-controller/cmd/mesh-controller/collection.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/novox/mesh-controller/internal/artifacts"
)

// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
// (novox/hq issue 253, ADR 0189).
//
// **The question to answer before the store's collector runs for real.** The collector deletes
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
// dry run go.
//
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
// new verb in the seat's row.

type collectionReport struct {
	// Store is the artifact store as this machine reached it; empty when it is not on the network.
	Store string `json:"store"`
	// KeptArchives is how many archives the mesh keeps, for either reason.
	KeptArchives int `json:"kept_archives"`
	// Held is how many of them the store holds by their manifest.
	Held int `json:"held"`
	// Unheld are the kept archives the collector would delete tonight if it ran for real.
	Unheld []string `json:"unheld"`
	// Missing are kept archives the store does not have at all.
	Missing []string `json:"missing"`
	// Unasked is how many could not be asked about, and why the asking stopped.
	Unasked int    `json:"unasked"`
	Stopped string `json:"stopped,omitempty"`
	// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
	// collected — split by kind.
	Eligible         int `json:"eligible"`
	EligibleImages   int `json:"eligible_images"`
	EligibleArchives int `json:"eligible_archives"`
	// SafeToCollect is whether every kept archive was asked about and every one is held.
	SafeToCollect bool `json:"safe_to_collect"`
}

func collectionCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("collection", flag.ContinueOnError)
	asJSON := set.Bool("json", false, "answer as JSON")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if len(positionals) != 0 {
		return errors.New("collection [--json]")
	}

	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	kept, err := inv.KeptArchives(ctx)
	if err != nil {
		return err
	}
	eligible, err := inv.ToCollect(ctx)
	if err != nil {
		return err
	}
	report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
	for _, reference := range eligible {
		if strings.Contains(reference, "/blobs/") {
			report.EligibleArchives++
		} else {
			report.EligibleImages++
		}
	}

	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return err
	}
	report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
	if err != nil {
		return err
	}
	if report.Store == "" {
		report.Unasked = len(kept)
		report.Stopped = "this mesh has no artifact store on its network"
	} else {
		report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
	}
	report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0

	if *asJSON {
		encoder := json.NewEncoder(os.Stdout)
		encoder.SetIndent("", "  ")
		return encoder.Encode(report)
	}
	printCollection(report)
	return nil
}

// askHeld asks the store about each kept archive, stopping at the first answer that is not about
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
// identical failures says less than one line.
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
	for i, reference := range kept {
		held, err := store.Held(ctx, reference)
		switch {
		case err == nil && held:
			report.Held++
		case err == nil:
			report.Unheld = append(report.Unheld, reference)
		case errors.Is(err, artifacts.Gone):
			report.Missing = append(report.Missing, reference)
		case errors.Is(err, artifacts.ErrNotOurs):
			// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
			report.Unasked++
		default:
			return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
		}
	}
	return report.Unasked, ""
}

func printCollection(r collectionReport) {
	store := r.Store
	if store == "" {
		store = "(not on the network)"
	}
	fmt.Printf("artifact store       %s\n", store)
	fmt.Printf("kept archives        %d\n", r.KeptArchives)
	fmt.Printf("  held               %d\n", r.Held)
	fmt.Printf("  unheld             %d\n", len(r.Unheld))
	fmt.Printf("  missing            %d\n", len(r.Missing))
	if r.Unasked > 0 {
		fmt.Printf("  not asked          %d (%s)\n", r.Unasked, r.Stopped)
	}
	fmt.Printf("eligible to let go   %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
	if len(r.Unheld) > 0 {
		fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
		for _, reference := range r.Unheld {
			fmt.Printf("  %s\n", reference)
		}
	}
	if len(r.Missing) > 0 {
		fmt.Println("\nmissing — kept by the mesh, not in the store:")
		for _, reference := range r.Missing {
			fmt.Printf("  %s\n", reference)
		}
	}
	fmt.Println()
	if r.SafeToCollect {
		fmt.Println("every kept archive is held: the store's collector may run for real")
	} else {
		fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
	}
}