Design 47 H1: indexer_take_in_again re-reads a repository at its kept merge

A repository with no merge since the walk of the module graph keeps its
modules' direct requirements only, and no tool took it in again. The tool
asks the indexer, through the state takeins, to read the repository's trees
at the merge its taken key keeps and apply them as a take-in does, whose
recompute of the modules state is the one a publish re-pin runs; kept
requirements and pins gain the transitive packages. An unknown repository,
or one never taken in, is refused in the answer and as the tool's error.
It keeps no catalog revision yet: H-A-211's revision follows with the
re-pin's.

Builds: design 47 H H-A-208, H-A-210, H-A-201 (hq a7b876c1, d7d18ef0)
This commit is contained in:
2026-11-20 23:16:25 +01:00
parent b0d2d22002
commit 7199824e9d
8 changed files with 185 additions and 2 deletions
@@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn {
return r
}
// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in
// again at a time beside the held ones.
func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn {
a.s.again.Lock()
defer a.s.again.Unlock()
return a.Indexer.TakeInAgain(ctx, repository)
}
// HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped.
func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn {
r := index.TakenIn{Module: branch}
@@ -24,6 +24,8 @@ type takerIn interface {
// HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744).
HeldAgain(ctx context.Context, branch string) index.TakenIn
HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn
// TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210).
TakeInAgain(ctx context.Context, repository string) index.TakenIn
}
// takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and
@@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Held != "" {
asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held)
a.Module = a.Held
} else if a.Again != "" {
asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again)
a.Module = a.Again
} else if a.Registry != nil {
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
} else if a.Recompute {
@@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error)
}
return a.Result, nil
}},
{Name: TakeInAgainTool,
Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " +
"module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " +
"package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " +
"fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.",
Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"},
"why": map[string]any{"type": "string", "description": "why it is taken in again now"}}},
Run: func(args map[string]any) (any, error) {
repository := str(args, "repository")
if repository == "" {
return nil, errors.New("name the repository (repository) as owner/name")
}
data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("again."+keys.EscapeName(repository), data)
if err != nil {
return nil, err
}
if a.Result != nil && !a.Result.Stored {
return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer)
}
return a.Result, nil
}},
{Name: TakeInSkipTool,
Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " +
"with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " +
@@ -0,0 +1,69 @@
package index
import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
)
// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the
// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since.
const TakeInAgainTool = "indexer_take_in_again"
// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every
// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute
// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there.
// It keeps no catalog revision and asks no gate (H-A-210). A repository with no taken key is refused, said in the answer.
func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn {
repository = strings.TrimSpace(repository)
r := TakenIn{Module: repository, Repository: repository}
refuse := func(format string, a ...any) TakenIn {
r.Answer = "refused: " + fmt.Sprintf(format, a...)
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
if ix.Taken == nil || ix.Modules == nil {
return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again")
}
key, err := trunk.RepositoryKey(repository)
if err != nil {
return refuse("%q names no repository as owner/name", repository)
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
return refuse("the state taken is not on the bus yet")
}
if err != nil {
return refuse("taken.%s cannot be read: %v", key, err)
}
if !found {
return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key)
}
t, err := takein.DecodeTaken(value)
if err != nil || t.Merge == "" {
return refuse("taken.%s names no merge commit", key)
}
ix.pendingMu.Lock() // one take-in step at a time (pending.go)
defer ix.pendingMu.Unlock()
f, err := ix.wholeAt(ctx, repository, t.Merge)
if err != nil {
return refuse("%v", err)
}
f.Branch = TrunkBranch
applied, err := ix.ApplyMessage(ctx, f)
if err != nil {
return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err)
}
r.Stored, r.Commit = true, t.Merge
r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written))
if len(applied.Written) > 0 {
r.Answer += " (" + strings.Join(applied.Written, ", ") + ")"
}
ix.logf("take-in again of %s: %s", repository, r.Answer)
return r
}
@@ -0,0 +1,72 @@
package index
import (
"context"
"maps"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct
// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the
// module's kept requirements and pins gain the transitive package. An unknown repository is refused.
func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) {
const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Taken = newMem()
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"}
walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"}
goMods := direct // before the rule: the client's go.mod reads as requiring nothing
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
if c, ok := goMods[dir]; ok {
return []byte(c), nil
}
if dir == "modules/c/client" {
return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil
}
return []byte(ptpGoMods[dir]), nil
}
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram},
map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}})
var readAt []string
ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) {
readAt = append(readAt, repository+"@"+sha)
return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil
}
ctx := context.Background()
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || slices.Contains(m.Inputs2.Packages, conditions) || strings.Contains(m.None2, conditions) {
t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", m.Inputs2, m.None2)
}
if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") {
t.Errorf("unknown repository: %+v, want refused", r)
}
goMods = maps.Clone(direct)
maps.Copy(goMods, walked)
r := ix.TakeInAgain(ctx, "novox/mesh-delivery")
if !r.Stored || r.Commit != merge {
t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge)
}
if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge {
t.Errorf("read at %v, want the kept merge commit", readAt)
}
m, _ := moduleOf(t, mods, "g")
if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) {
t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions)
}
// The recompute pins it: nothing is published here, so the fingerprint waits on it by name (H-A-161), and the pins the
// entry names from the kept requirements hold it with no version until it is (H-A-204).
if !strings.Contains(m.None2, conditions) {
t.Errorf("after: the recompute's reason %q, want it waiting on %s", m.None2, conditions)
}
if v, named := releasedPins(nil, nil, catalogue.Manifest{}, "novox/mesh-delivery", m.Inputs2.Packages)[conditions]; !named || v != "" {
t.Errorf("after: pins name %s %v (%q), want it named, waiting", conditions, named, v)
}
}
@@ -46,6 +46,8 @@ type TakeInAsk struct {
HeldSource int64 `json:"held-source,omitempty"`
Skip bool `json:"skip,omitempty"`
NamedMember string `json:"named-member,omitempty"`
// Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210).
Again string `json:"again,omitempty"`
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
RunDrop string `json:"run-drop,omitempty"`
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
@@ -206,8 +206,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") {
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -67,6 +67,7 @@
"indexer_pin_drop",
"indexer_pending",
"indexer_take_in_pending",
"indexer_take_in_again",
"indexer_take_in_skip",
"indexer_runs",
"indexer_run_drop"