node-engine: compose-candidate reads a branch pipeline's catalog revision through mesh-catalog.revision and composes it as the candidate (A-226b)
Builds: design 47 G (G-22 lead answers 2) A-226b; ADR 0593 §7
This commit is contained in:
@@ -9,6 +9,8 @@ import (
|
|||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||||
"github.com/novox/mesh-host/internal/buckets"
|
"github.com/novox/mesh-host/internal/buckets"
|
||||||
engcomp "github.com/novox/mesh-host/internal/compositions"
|
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||||
"maps"
|
"maps"
|
||||||
@@ -143,35 +145,55 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi
|
|||||||
a.Verdict = nec.Unready
|
a.Verdict = nec.Unready
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
group, heads, err := nec.ParseCompositionKey(ask.Candidate)
|
key, err := nec.ParseCandidateKey(ask.Candidate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return refuse(err.Error())
|
return refuse(err.Error())
|
||||||
}
|
}
|
||||||
// The ask names the check (the composition key, `<group>.<heads hash>`); the indexer keys the group's candidate by
|
if key.Kind == nec.CandidateRun {
|
||||||
// the group alone (buckets.CandidateKey) and holds its heads hash in the value.
|
// A run on request's candidate is named by `run-<id>` (A-226b); the node-engine reads none yet.
|
||||||
ckey, err := buckets.CandidateKey(group)
|
return refuse("the node-engine reads no run on request's candidate yet: " + ask.Candidate)
|
||||||
if err != nil {
|
|
||||||
return refuse(err.Error())
|
|
||||||
}
|
}
|
||||||
if !s.ready.Load() {
|
if !s.ready.Load() {
|
||||||
return unready(errNotReady.Error())
|
return unready(errNotReady.Error())
|
||||||
}
|
}
|
||||||
base := run.add(c.scratch())
|
base := run.add(c.scratch())
|
||||||
raw, err := base.get(ctx, buckets.Candidates, ckey)
|
var cand meshcatalog.CandidatesByGroupValue
|
||||||
if err != nil {
|
var group, heads string
|
||||||
return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
|
if key.Kind == nec.CandidateRevision {
|
||||||
}
|
// A branch pipeline's catalog graph revision (A-226b): its entries are read through the seat mesh-catalog's verb
|
||||||
if raw == nil {
|
// revision and composed as the candidate, each module at the revision's fingerprint.
|
||||||
return refuse("the indexer holds no candidate " + ask.Candidate)
|
var answered bool
|
||||||
}
|
cand, answered, err = s.revisionCandidate(ctx, base, key)
|
||||||
cand, err := meshcatalog.DecodeCandidatesByGroupValue(raw)
|
if err != nil {
|
||||||
if err != nil {
|
return unready(err.Error())
|
||||||
return refuse(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
|
}
|
||||||
}
|
if !answered {
|
||||||
if cand.Heads != heads {
|
return refuse(fmt.Sprintf("the catalog holds no revision %d of the branch %s", key.Revision, key.Branch))
|
||||||
// The group moved to other heads: the check asked is replaced, and this node composes not a candidate it was
|
}
|
||||||
// not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads.
|
} else {
|
||||||
return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.Heads, heads))
|
group, heads = key.Group, key.HeadsHash
|
||||||
|
// The ask names the check (the composition key, `<group>.<heads hash>`); the indexer keys the group's candidate by
|
||||||
|
// the group alone (buckets.CandidateKey) and holds its heads hash in the value.
|
||||||
|
ckey, err := buckets.CandidateKey(group)
|
||||||
|
if err != nil {
|
||||||
|
return refuse(err.Error())
|
||||||
|
}
|
||||||
|
raw, err := base.get(ctx, buckets.Candidates, ckey)
|
||||||
|
if err != nil {
|
||||||
|
return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
|
||||||
|
}
|
||||||
|
if raw == nil {
|
||||||
|
return refuse("the indexer holds no candidate " + ask.Candidate)
|
||||||
|
}
|
||||||
|
cand, err = meshcatalog.DecodeCandidatesByGroupValue(raw)
|
||||||
|
if err != nil {
|
||||||
|
return refuse(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err))
|
||||||
|
}
|
||||||
|
if cand.Heads != heads {
|
||||||
|
// The group moved to other heads: the check asked is replaced, and this node composes not a candidate it was
|
||||||
|
// not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads.
|
||||||
|
return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.Heads, heads))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Every module's manifest is read before either composition, several at once, into the shadow's own: a composition
|
// Every module's manifest is read before either composition, several at once, into the shadow's own: a composition
|
||||||
// reads every manifest in the mesh one by one, which off the control-node took about 35 s each (issue 1126).
|
// reads every manifest in the mesh one by one, which off the control-node took about 35 s each (issue 1126).
|
||||||
@@ -481,3 +503,69 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed
|
|||||||
// candidateNoneNoModule is the words the indexer writes in a candidate's module that the merge preview names no manifest
|
// candidateNoneNoModule is the words the indexer writes in a candidate's module that the merge preview names no manifest
|
||||||
// for: the change removes the module. It is the indexer's text, kept here until the catalogue client names it.
|
// for: the change removes the module. It is the indexer's text, kept here until the catalogue client names it.
|
||||||
const candidateNoneNoModule = "the merge preview names no manifest for the module's directory"
|
const candidateNoneNoModule = "the merge preview names no manifest for the module's directory"
|
||||||
|
|
||||||
|
// revisionCandidate reads the catalog graph revision key names, whole, page by page, through the seat mesh-catalog's verb
|
||||||
|
// revision (the Entry call, the engine's own name), and makes the candidate it composes as: one pull per repository and
|
||||||
|
// commit with each module's fingerprint and manifest digest, and a module the base holds that the revision does not marked as
|
||||||
|
// removed. answered is false when the catalog holds no such revision.
|
||||||
|
func (s *Shadow) revisionCandidate(ctx context.Context, base *Composer, key nec.Candidate) (cand meshcatalog.CandidatesByGroupValue, answered bool, err error) {
|
||||||
|
call := base.env.Entry
|
||||||
|
if call == nil {
|
||||||
|
return cand, false, errors.New("this node's engine cannot ask the catalog for a revision")
|
||||||
|
}
|
||||||
|
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(key.Branch), Revision: key.Revision}
|
||||||
|
type group struct{ repository, commit string }
|
||||||
|
pulls := map[group]map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for {
|
||||||
|
raw, err := call(meshcatalogclient.RevisionVerb, in)
|
||||||
|
if err != nil {
|
||||||
|
return cand, false, fmt.Errorf("the catalog's revision %d of %s cannot be read: %w", key.Revision, key.Branch, err)
|
||||||
|
}
|
||||||
|
body, said, ok := buscalls.Unwrap(raw)
|
||||||
|
if !ok {
|
||||||
|
return cand, false, fmt.Errorf("the catalog's revision %d of %s cannot be read: %s", key.Revision, key.Branch, said)
|
||||||
|
}
|
||||||
|
var a meshcatalog.RevisionAnswer
|
||||||
|
if err := json.Unmarshal(body, &a); err != nil {
|
||||||
|
return cand, false, fmt.Errorf("the catalog's revision %d of %s is not a revision answer: %w", key.Revision, key.Branch, err)
|
||||||
|
}
|
||||||
|
if a.NoneYet || a.Revision == nil || a.Revision.Revision != key.Revision {
|
||||||
|
return cand, false, nil
|
||||||
|
}
|
||||||
|
for _, e := range a.Entries {
|
||||||
|
seen[e.Module] = true
|
||||||
|
if e.Fp == "" {
|
||||||
|
continue // no build at the revision: composed as the trunk holds it
|
||||||
|
}
|
||||||
|
g := group{e.Repository, e.Commit}
|
||||||
|
if pulls[g] == nil {
|
||||||
|
pulls[g] = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
||||||
|
}
|
||||||
|
pulls[g][e.Module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: e.Fp, Manifest: e.ManifestDigest}
|
||||||
|
}
|
||||||
|
if a.Next == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
in.After = a.Next
|
||||||
|
}
|
||||||
|
removed := map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
||||||
|
base.mu.Lock()
|
||||||
|
for module := range base.modules {
|
||||||
|
if !seen[module] {
|
||||||
|
removed[module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{None2: candidateNoneNoModule}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
base.mu.Unlock()
|
||||||
|
cand.Group = key.Branch
|
||||||
|
for g, fps := range pulls {
|
||||||
|
cand.Pulls = append(cand.Pulls, meshcatalog.CandidatesByGroupValuePullsItem{Repository: g.repository, Head: g.commit, Fps: fps})
|
||||||
|
}
|
||||||
|
sort.Slice(cand.Pulls, func(i, j int) bool {
|
||||||
|
return cand.Pulls[i].Repository+cand.Pulls[i].Head < cand.Pulls[j].Repository+cand.Pulls[j].Head
|
||||||
|
})
|
||||||
|
if len(removed) > 0 {
|
||||||
|
cand.Pulls = append(cand.Pulls, meshcatalog.CandidatesByGroupValuePullsItem{Fps: removed})
|
||||||
|
}
|
||||||
|
return cand, true, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -85,7 +85,18 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", ManifestDigest: digest}
|
entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", ManifestDigest: digest}
|
||||||
env.Entry = EntryCall(func(_ context.Context, _, _ string, _ any) (json.RawMessage, error) {
|
// A-226b: the revision a branch pipeline names is read through the verb revision: revision 362 of main holds lemurs at its
|
||||||
|
// new build and manifest; no other revision is held.
|
||||||
|
env.Entry = EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) {
|
||||||
|
if verb == meshcatalog.VerbRevision {
|
||||||
|
in, _ := args.(meshcatalog.RevisionInput)
|
||||||
|
if in.Revision != 362 || in.Branch != "main" {
|
||||||
|
return json.Marshal(meshcatalog.RevisionAnswer{NoneYet: true})
|
||||||
|
}
|
||||||
|
moved := entry
|
||||||
|
moved.Fp, moved.ManifestDigest = fpNew, digest2
|
||||||
|
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Branch: "main", Revision: 362}, Entries: []meshcatalog.CatalogEntry{moved}})
|
||||||
|
}
|
||||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
||||||
})
|
})
|
||||||
var handed atomic.Int64
|
var handed atomic.Int64
|
||||||
@@ -141,6 +152,21 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) {
|
|||||||
if _, err := nec.EncodeAnswer(r); err != nil {
|
if _, err := nec.EncodeAnswer(r); err != nil {
|
||||||
t.Errorf("the refused answer does not fit the seat's shape: %v", err)
|
t.Errorf("the refused answer does not fit the seat's shape: %v", err)
|
||||||
}
|
}
|
||||||
|
// A-226b: a branch pipeline's revision composes as the candidate, each module at the revision's build; one the catalog does
|
||||||
|
// not hold, and a run on request's key, are refused in words.
|
||||||
|
revKey, _ := nec.RevisionKey("main", 362)
|
||||||
|
rv := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: revKey})
|
||||||
|
if rv.Verdict != nec.Composed || len(rv.Modules) != 1 || rv.Modules[0].Fp != fpNew || !rv.Modules[0].Changed {
|
||||||
|
t.Errorf("a catalog revision answered %+v", rv)
|
||||||
|
}
|
||||||
|
noRev, _ := nec.RevisionKey("main", 999)
|
||||||
|
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: noRev}); m.Verdict != nec.Refused || !strings.Contains(m.Refused, "holds no revision 999") {
|
||||||
|
t.Errorf("a revision the catalog does not hold answered %+v", m)
|
||||||
|
}
|
||||||
|
runKey, _ := nec.RunKey("r1")
|
||||||
|
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: runKey}); m.Verdict != nec.Refused || !strings.Contains(m.Refused, "run on request") {
|
||||||
|
t.Errorf("a run's key answered %+v", m)
|
||||||
|
}
|
||||||
missing, _ := nec.CompositionKey("novox/x#8", "0123456789ab")
|
missing, _ := nec.CompositionKey("novox/x#8", "0123456789ab")
|
||||||
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") {
|
if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") {
|
||||||
t.Errorf("a candidate the indexer does not hold answered %+v", m)
|
t.Errorf("a candidate the indexer does not hold answered %+v", m)
|
||||||
|
|||||||
Reference in New Issue
Block a user