builder, assigner: say why a package-published is not said and why an added module carries no parts
Builds: design 47 G §588, §52, §334, A-73, A-316
This commit is contained in:
@@ -30,10 +30,12 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
}
|
||||
fp := p.w.trunkOf(branch)[module].Fp2
|
||||
if fp == "" {
|
||||
p.skippedAdd(node, module, "", "the branch "+branch+" names no fingerprint of it")
|
||||
continue
|
||||
}
|
||||
k, ok := p.namedBuild(module, fp)(fp)
|
||||
if !ok {
|
||||
p.skippedAdd(node, module, fp, "no version-released of it is heard for that fingerprint, so the candidate carries no version and no parts")
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
@@ -48,3 +50,18 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
|
||||
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
|
||||
}
|
||||
|
||||
// skippedAdd says once per node, module and fingerprint why a module the candidate adds is not listed in `added`.
|
||||
func (p *passer) skippedAdd(node, module, fp, why string) {
|
||||
key := node + "/" + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the candidate adds %s to %s with no parts: %s (fingerprint %q)", module, node, why, fp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ type passer struct {
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(oc.Condition) error
|
||||
said map[string]bool
|
||||
// skipped are the added modules whose absence from a candidate's added was said (skippedAdd), once each.
|
||||
skipped map[string]bool
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module a candidate adds with no release heard is said once, not each pass.
|
||||
func TestASkippedAddIsSaidOncePerModuleAndFingerprint(t *testing.T) {
|
||||
var logs []string
|
||||
p := &passer{logf: func(f string, a ...any) { logs = append(logs, strings.TrimSpace(f)) }}
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp2", "why")
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("said %d times, want 2: %v", len(logs), logs)
|
||||
}
|
||||
}
|
||||
@@ -210,13 +210,19 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
|
||||
if err != nil {
|
||||
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
|
||||
}
|
||||
if len(marked) == 0 {
|
||||
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o))
|
||||
}
|
||||
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
|
||||
for _, m := range marked {
|
||||
if rv.say == nil {
|
||||
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
|
||||
continue
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
} else {
|
||||
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, o.Digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -261,3 +267,26 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests.
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
|
||||
version := strings.TrimPrefix(o.Version, "v")
|
||||
if version == "" || o.Digest == "" {
|
||||
return "it names no version or no digest (a by-digest publish)"
|
||||
}
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
return "the reservations cannot be read: " + err.Error()
|
||||
}
|
||||
var out []string
|
||||
for _, v := range all {
|
||||
if v.Version != version {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests marks nothing)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "no reservation holds version " + version
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -201,3 +202,24 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
}
|
||||
|
||||
// A published heard before the build recorded its digests marks nothing and says why, so a package-published never said is
|
||||
// readable in the journal.
|
||||
func TestAPublishedBeforeTheDigestsIsSaidAsMarkingNothing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var logs []string
|
||||
logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
|
||||
none := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf)
|
||||
rv.filled.Store(true)
|
||||
if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.1", Digest: "sha256:a"})
|
||||
if !strings.Contains(strings.Join(logs, "\n"), "package-published not said") || !strings.Contains(strings.Join(logs, "\n"), "marks no artifact") {
|
||||
t.Fatalf("log = %q, want package-published not said", logs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,6 +311,7 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
break
|
||||
}
|
||||
if v.Fingerprint == "" || v.BuiltAt.IsZero() {
|
||||
r.logf("publish again: %s %s is not queued: it holds no fingerprint or its build has not ended with every digest", v.Module, v.Version)
|
||||
continue
|
||||
}
|
||||
e, err := r.Entry(ctx, v.Fingerprint)
|
||||
@@ -320,20 +321,28 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
byHolder := map[string]map[string]string{}
|
||||
for _, ref := range v.Missing() {
|
||||
if _, no := v.Unstored[ref]; no {
|
||||
r.logf("publish again: %s %s: %s is not queued: a holder refused it as not stored", v.Module, v.Version, ref)
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range e.Artifacts {
|
||||
seat, ok := SeatOfArtifact(a)
|
||||
if a.Ref != ref || !ok {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if hs := holdersOf(seat); len(hs) > 0 {
|
||||
if byHolder[hs[0]] == nil {
|
||||
byHolder[hs[0]] = map[string]string{}
|
||||
}
|
||||
byHolder[hs[0]][ref] = v.Artifacts[ref]
|
||||
} else {
|
||||
r.logf("publish again: %s %s: %s is not queued: no module running that claims the registry seat %s is known to the builder since its start", v.Module, v.Version, ref, seat)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
r.logf("publish again: %s %s: %s is not queued: the catalog entry %s names no registry seat for it", v.Module, v.Version, ref, v.Fingerprint)
|
||||
}
|
||||
}
|
||||
did := false
|
||||
for holder, arts := range byHolder {
|
||||
@@ -361,6 +370,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
|
||||
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
|
||||
return false, err
|
||||
} else if found {
|
||||
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
|
||||
return false, nil
|
||||
}
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt()})
|
||||
@@ -380,6 +390,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
|
||||
}
|
||||
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts", v.Module, v.Version, holder, len(arts))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -607,7 +618,11 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
||||
Commit: v.Commit, Transfer: "republish-" + q.Module + "-" + v.Version + "-" + strings.TrimPrefix(ref, q.Module+":")})
|
||||
cancel()
|
||||
if err == nil {
|
||||
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil {
|
||||
|
||||
Reference in New Issue
Block a user