diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index cae5d361..53bffac7 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -72,6 +72,10 @@ type Change struct { Approved string `json:"approved,omitempty"` At time.Time `json:"at"` Moved []Moved `json:"moved"` + // Pull and Head name the pull request a value is held for, dropped from, or applied from at its merge (novox/hq + // ADR 0502): the actions hold and drop are always about one, and a set with a pull is its merge. + Pull string `json:"pull,omitempty"` + Head string `json:"head,omitempty"` } // Config is the module: its store, its clock, and one lock, so a set and a resolution never interleave. @@ -86,6 +90,8 @@ type Config struct { // Asks is the operator channel as this module asks on it (proposals.go); nil, nothing can be proposed. Asks AskBus + // Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone. + Forge PullForge // RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often // it looks; zero is RouterAnswersWithin and RouterAnswersEvery. RouterWithin, RouterEvery time.Duration @@ -109,6 +115,13 @@ type Call struct { // Clear asks a proposal to clear the value instead of setting it; Proposal names one proposal to read. Clear bool Proposal string + // Pull names the open pull request that introduces the key, for propose (a held value) and pull-declares; Head is + // the commit its declarations are read at; Declares the config declarations the head adds or changes, by module; + // Pulls the pull requests held asks about, each at a head. + Pull string + Head string + Declares map[string]any + Pulls []PullAt // approval is the operator's warrant a set or a clear is made on: given by a decided proposal alone, never by // a call's arguments. @@ -149,8 +162,13 @@ func (c *Config) Do(call Call) (any, error) { return c.Propose(call) case "proposals": return c.Proposals(call) + case "pull-declares": + return c.PullDeclares(call) + case "held": + return c.Held(call) } - return nil, fmt.Errorf("config does set, clear, list, history, propose or proposals; %q is none of them", call.Action) + return nil, fmt.Errorf("config does set, clear, list, history, propose, proposals, pull-declares or held; %q is "+ + "none of them", call.Action) } // level is the level a call names, refused as ADR 0348 §3 says: a module's own value is for one node. @@ -178,6 +196,10 @@ func (c *Config) Change(call Call) (*SetAnswer, error) { if call.Clear || call.Proposal != "" { return nil, fmt.Errorf("clear and proposal are a proposal's: %s takes neither", call.Action) } + if call.Pull != "" { + return nil, fmt.Errorf("pull names the pull request a value is held for, which only a proposal does: %s takes "+ + "none", call.Action) + } call.approval = nil // only a decided proposal gives one c.mu.Lock() defer c.mu.Unlock() diff --git a/modules/mesh-config/cmd/mesh-config/list.go b/modules/mesh-config/cmd/mesh-config/list.go index a2169023..ba129102 100644 --- a/modules/mesh-config/cmd/mesh-config/list.go +++ b/modules/mesh-config/cmd/mesh-config/list.go @@ -17,6 +17,8 @@ type Listed struct { Predefined map[string]Published `json:"predefined,omitempty"` // Withheld is why the newest list of predefined keys is not published, when it is not. Withheld string `json:"withheld,omitempty"` + // Held is the approved values held for open pull requests, not yet set (novox/hq ADR 0502). + Held []HeldValue `json:"held,omitempty"` } // SetValue is one value set, at its level. @@ -85,6 +87,10 @@ func (c *Config) List(call Call) (*Listed, error) { if call.Module == "" { out.Predefined = s.Predefined } + out.Held = heldListed(s, call) + if len(out.Held) == 0 { + out.Held = nil + } return out, nil } diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index c915d3ef..1334b097 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -94,7 +94,7 @@ func (runtimeAsks) Warranted(ask string) error { } func main() { - c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}} + c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}} go resolving(c) go hearing(c) if err := stdio.Serve("", Tools(c)); err != nil { @@ -130,7 +130,14 @@ func resolving(c *Config) { tick := time.NewTicker(5 * time.Minute) defer tick.Stop() time.Sleep(time.Second) - for { + for sweep := true; ; { + if sweep { + // A merge or a close of a pull request a value is held for, heard nowhere, is found here (ADR 0502). + for _, s := range c.SweepPulls() { + logf("[mesh-config] %s", s) + } + } + sweep = false r, err := c.Reconcile() switch { case err != nil: @@ -149,6 +156,7 @@ func resolving(c *Config) { default: } case <-tick.C: + sweep = true } } } @@ -175,6 +183,28 @@ func hearing(c *Config) { logf("[mesh-config] does not hear the operator's answers yet (%v); asking again in %s", err, wait) } }() + for _, pattern := range []string{"gitea.pull.merged", "gitea.pull.closed", "gitea.forge.pull-request.closed"} { + go func(pattern string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { + var p PullEvent + if json.Unmarshal(e.Body, &p) != nil { + logf("[mesh-config] %s is not readable", pattern) + return nil + } + for _, s := range c.HeardPull(p, pattern == "gitea.pull.merged") { + logf("[mesh-config] %s", s) + } + return nil + }) + if err == nil { + return + } + logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait) + } + }(pattern) + } tick := time.NewTicker(time.Minute) defer tick.Stop() for range tick.C { @@ -199,10 +229,13 @@ func Tools(c *Config) []stdio.Tool { "who and why. A trusted value is set only at the controller's terminal or on the operator's Approve: " + "propose asks the operator, under mesh-config's own name on a channel that proves who answered, and sets " + "the value on Approve if what it was shown against still stands; proposals lists them, or one with " + - "proposal. list and history are narrowed by module, node and key.", + "proposal. A proposal naming the open pull request that introduces its key (pull) is judged against what " + + "that pull request declares and asked of the operator, and on Approve is held until the pull request " + + "merges, then set; list shows the held values. pull-declares and held are the controller's. list and " + + "history are narrowed by module, node and key.", Input: map[string]any{"type": "object", "properties": map[string]any{ "action": map[string]any{"type": "string", - "enum": []string{"list", "set", "clear", "history", "propose", "proposals"}, + "enum": []string{"list", "set", "clear", "history", "propose", "proposals", "pull-declares", "held"}, "description": "list when not given"}, "key": map[string]any{"type": "string", "description": "the config key"}, "value": map[string]any{"description": "for set: the value, of the key's declared type"}, @@ -215,6 +248,14 @@ func Tools(c *Config) []stdio.Tool { "clear": map[string]any{"type": "boolean", "description": "for propose: propose clearing the value at its level instead of setting one"}, "proposal": map[string]any{"type": "string", "description": "for proposals: one proposal's id, read whole"}, + "pull": map[string]any{"type": "string", "description": "owner/repository#number: for propose, the open " + + "pull request that introduces the key, so the value is judged against what it declares and held until " + + "it merges (a module's own value on one node); for pull-declares, the pull request declared for"}, + "head": map[string]any{"type": "string", "description": "for pull-declares: the commit the declarations are read at"}, + "declares": map[string]any{"type": "object", "description": "for pull-declares (the controller's): the " + + "config declarations the head adds or changes against its base, by module and key"}, + "pulls": map[string]any{"type": "array", "description": "for held (the controller's): the pull requests " + + "asked about, each {pull, head}", "items": map[string]any{"type": "object"}}, }}, Run: func(args map[string]any) (any, error) { call, err := callOf(args) @@ -230,7 +271,8 @@ func Tools(c *Config) []stdio.Tool { // callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. func callOf(args map[string]any) (Call, error) { known := map[string]bool{"action": true, "key": true, "value": true, "module": true, "node": true, "why": true, - "by": true, "limit": true, "clear": true, "proposal": true} + "by": true, "limit": true, "clear": true, "proposal": true, "pull": true, "head": true, "declares": true, + "pulls": true} var unknown []string for k := range args { if !known[k] { @@ -239,7 +281,7 @@ func callOf(args map[string]any) (Call, error) { } if len(unknown) > 0 { return Call{}, fmt.Errorf("config takes no %s; it takes action, key, value, module, node, why, by, limit, "+ - "clear and proposal", + "clear, proposal, pull, head, declares and pulls", strings.Join(unknown, ", ")) } str := func(k string) (string, error) { @@ -256,7 +298,8 @@ func callOf(args map[string]any) (Call, error) { var call Call var problems []error for k, into := range map[string]*string{"action": &call.Action, "key": &call.Key, "module": &call.Module, - "node": &call.Node, "why": &call.Why, "by": &call.By, "proposal": &call.Proposal} { + "node": &call.Node, "why": &call.Why, "by": &call.By, "proposal": &call.Proposal, "pull": &call.Pull, + "head": &call.Head} { s, err := str(k) problems = append(problems, err) *into = s @@ -275,6 +318,19 @@ func callOf(args map[string]any) (Call, error) { } call.Clear = b } + if v, ok := args["declares"]; ok && v != nil { + m, ok := v.(map[string]any) + if !ok { + problems = append(problems, errors.New("declares is an object: a module to its keys' declarations")) + } + call.Declares = m + } + if v, ok := args["pulls"]; ok && v != nil { + raw, _ := json.Marshal(v) + if err := json.Unmarshal(raw, &call.Pulls); err != nil { + problems = append(problems, errors.New("pulls is a list of {pull, head}")) + } + } if v, ok := args["value"]; ok { // The value as given, a JSON value: a number for a number, never its text; the validation judges it. call.Value = v diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index 84d53a09..d6cb304d 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -97,6 +97,11 @@ type Proposal struct { HadValue bool `json:"had-value"` WasDigest string `json:"was-digest"` Trusted bool `json:"trusted,omitempty"` + // Pull is the open pull request the value is held for, when it is (novox/hq ADR 0502): the value is then not set + // on Approve but kept apart until the pull request merges, and DeclDigest is the digest of the pull request's + // declaration of the key it was judged and shown against. + Pull string `json:"pull,omitempty"` + DeclDigest string `json:"declaration,omitempty"` // Reaches is each module on each node whose value it moves, as it was judged when proposed. Reaches []string `json:"reaches"` // From is who proposed it, as the bus proved the caller; Why is theirs; At is when. @@ -168,15 +173,24 @@ func (p Proposal) Fingerprint() string { // about is what its ask is about: the value's place, so a newer proposal for it replaces an older one still open // (ADR 0277 §10, at a level, a key). -func (p Proposal) about() string { return "config." + p.Level.ValueKey(p.Key) } +func (p Proposal) about() string { + if p.Pull != "" { + return "config.held." + pullKey(p.Pull) + "." + p.Level.ValueKey(p.Key) + } + return "config." + p.Level.ValueKey(p.Key) +} // Act is what an answer binds: this proposal's level, key, the digest of its exact value and of the value it was // shown against, who proposed it and when, and the answer. func (p Proposal) Act(option string) asks.Act { - return asks.Act{"verb": ProposalVerb, "option": option, "proposal": p.ID, "action": p.Action, + act := asks.Act{"verb": ProposalVerb, "option": option, "proposal": p.ID, "action": p.Action, "level": p.Level.Kind, "node": p.Level.Node, "module": p.Level.Module, "key": p.Key, "value": p.Digest, "was": p.WasDigest, "had-value": strconv.FormatBool(p.HadValue), "from": p.From, "at": p.At.UTC().Format(time.RFC3339Nano)} + if p.Pull != "" { + act["pull"], act["declaration"] = p.Pull, p.DeclDigest + } + return act } // ---- what the operator reads -------------------------------------------------------------------------- @@ -255,6 +269,16 @@ func (p Proposal) summary() string { if p.Trusted { trusted = ", a trusted value," } + if p.Pull != "" { + s := fmt.Sprintf("Holds %s for %s on %s for %s; approved, it takes effect when that pull request merges, "+ + "and is dropped if it closes unmerged. Nothing reaches a machine before the merge.", p.Key, p.Level.Module, + p.Level.Node, p.Pull) + if len([]rune(s)) > asks.SummaryLength { + s = fmt.Sprintf("Holds %s for %s on %s for %s until it merges; dropped if it closes unmerged.", p.Key, + p.Level.Module, p.Level.Node, p.Pull) + } + return s + } reach := p.reaches() s := fmt.Sprintf("%s%s at %s; approved, it takes effect at once on every node it reaches: %s.", verb, trusted, p.Level, reach) @@ -280,6 +304,9 @@ func (p Proposal) headline() string { if p.Action == "clear" { verb = "Clear" } + if p.Pull != "" { + verb = "Hold" + } where := "for the whole mesh" switch p.Level.Kind { case LevelNode: @@ -331,6 +358,9 @@ func (p Proposal) Ask() asks.Ask { a.WholeDetails = w } does := "the value is set at once, and every node it reaches takes it" + if p.Pull != "" { + does = "the value is held for " + p.Pull + ": it is set when that pull request merges, and dropped if it closes unmerged" + } if p.Action == "clear" { does = "the value is cleared at once, and every node it reaches takes what is below it" } @@ -361,6 +391,8 @@ func (p Proposal) problem() string { return "its value is not the value its digest names: it changed after it was asked" case p.Action == "clear" && (len(p.Value) != 0 || p.Digest != valueDigest(nil, false)): return "a clear carries no value" + case p.Pull != "" && (p.Action != "set" || p.Level.Kind != LevelModule || p.DeclDigest == ""): + return "a value held for a pull request is a set of a module's own value, judged against its declaration" case valueDigest(p.Was, p.HadValue) != p.WasDigest: return "the value it was shown against is not the one its digest names: it changed after it was asked" } @@ -517,16 +549,41 @@ func (c *Config) propose(call Call, action string) (Proposal, asks.Ask, []string return Proposal{}, asks.Ask{}, nil, err } // **Judged as a set judges it, before anyone is asked** (ADR 0277 §4): no ask for a value the verb would refuse. - r, trusted, err := c.declared(s, level, call.Key) - if err != nil { - return fail("%v", err) - } + var r rule + var trusted bool + var declDigest string vk := level.ValueKey(call.Key) was, had := s.Values[vk] + if call.Pull != "" { + // **A value held for the open pull request that introduces its key** (novox/hq ADR 0502): judged against what + // the pull request declares at its head, as the controller told it, and asked of the operator whatever the + // key's trust. + if action != "set" { + return fail("a clear is not held for a pull request: only a value to set is") + } + if _, _, _, err := parsePull(call.Pull); err != nil { + return fail("%v", err) + } + call.Pull = strings.TrimSpace(call.Pull) + var herr error + r, _, declDigest, herr = heldRule(s, call, level) + if herr != nil { + return fail("%v", herr) + } + trusted = false + k, kept := s.Kept[keptKey(call.Pull, level.Node, level.Module, call.Key)] + was, had = Value{Value: k.Value}, kept + } else { + var derr error + r, trusted, derr = c.declared(s, level, call.Key) + if derr != nil { + return fail("%v", derr) + } + } now := c.now() prop := Proposal{ID: newProposalID(), Action: action, Level: level, Key: call.Key, HadValue: had, WasDigest: valueDigest(was.Value, had), Trusted: trusted, From: call.Caller, Why: why, At: now, - State: ProposalWaiting, Digest: valueDigest(nil, false)} + State: ProposalWaiting, Digest: valueDigest(nil, false), Pull: call.Pull, DeclDigest: declDigest} if had { prop.Was = compact(was.Value) } @@ -553,15 +610,19 @@ func (c *Config) propose(call Call, action string) (Proposal, asks.Ask, []string if had && string(compact(was.Value)) == string(prop.Value) { return fail("%s at %s already holds exactly %s", call.Key, level, prop.Value) } - after.Values[vk] = Value{Value: prop.Value} + if call.Pull == "" { + after.Values[vk] = Value{Value: prop.Value} + } case "clear": if !had { return fail("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) } delete(after.Values, vk) } - for _, m := range diff(Resolve(s), Resolve(after)) { - prop.Reaches = append(prop.Reaches, m.Module+" on "+m.Node) + if call.Pull == "" { + for _, m := range diff(Resolve(s), Resolve(after)) { + prop.Reaches = append(prop.Reaches, m.Module+" on "+m.Node) + } } if prop.Reaches == nil { prop.Reaches = []string{} @@ -763,6 +824,14 @@ func (c *Config) decide(p Proposal, w asks.Warrant) string { return fmt.Sprintf("proposal %s: approved by the operator, and not set: the decision could not be kept (%v); "+ "it is read again on the next look", p.ID, err) } + if p.Pull != "" { + // **A held value is kept apart, never set on Approve** (ADR 0502 clause 3). + said, err := c.keep(p) + if err != nil { + return keep(ProposalRefused, "approved by the operator, and refused on Approve, so nothing is held: "+err.Error()) + } + return keep(ProposalApproved, "approved by the operator and held: "+said) + } call := Call{Action: p.Action, Key: p.Key, Module: p.Level.Module, Node: p.Level.Node, Why: p.Why, Caller: p.From, approval: &approval{Proposal: p.ID, By: "the operator", Says: p.Approved, Was: p.Was, HadValue: p.HadValue, WasDigest: p.WasDigest}} @@ -865,6 +934,8 @@ type ProposalRow struct { Expires time.Time `json:"expires"` Fingerprint string `json:"fingerprint"` Approved string `json:"approved,omitempty"` + // Pull is the pull request a value is held for, when it is. + Pull string `json:"pull,omitempty"` // What one proposal read whole says besides. Value json.RawMessage `json:"value,omitempty"` Was json.RawMessage `json:"was,omitempty"` @@ -893,7 +964,8 @@ func (c *Config) Proposals(call Call) ([]ProposalRow, error) { continue } row := ProposalRow{ID: p.ID, Action: p.Action, Level: p.Level, Key: p.Key, State: p.State, Outcome: p.Outcome, - From: p.From, At: p.At, Expires: p.At.Add(ProposalLasts), Fingerprint: p.Fingerprint(), Approved: p.Approved} + From: p.From, At: p.At, Expires: p.At.Add(ProposalLasts), Fingerprint: p.Fingerprint(), Approved: p.Approved, + Pull: p.Pull} if p.State == ProposalWaiting && c.now().After(row.Expires) { row.State = "expired, not yet heard" } diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go new file mode 100644 index 00000000..5f7742a5 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -0,0 +1,629 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826). +// +// A key a pull request adds is declared by no registered manifest until it merges, so a value for it, which the +// pull request's composed check needs, was refused. The controller reads each pull request's manifests from the +// forge at its head and tells this module the config declarations the head adds or changes (PullDeclares); mesh-config +// reads no manifest itself (ADR 0495 §1). A proposal that names the pull request is judged against those, asked of the +// operator whatever the key's trust, and on Approve kept apart (KeptState) from the values it resolves. The +// controller asks for the approved held values of a check's pull requests (Held) and hands them to the composed +// check. At the merge the value is written as the module's own value on its node (PullMerged), where it resolves +// once the trunk declares the key; a pull request closed unmerged drops it (PullClosed). Nothing reaches a machine +// before the merge. + +// The states of held values, and the actions their changes carry in the history and in `config-changed`. +const ( + PullsState = "pulls" + KeptState = "kept" + + ActionHold = "hold" + ActionDrop = "drop" +) + +// pullName is a pull request as it is named: owner/repository#number. +var pullName = regexp.MustCompile(`^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)#([0-9]+)$`) + +func parsePull(s string) (owner, repo string, number int, err error) { + m := pullName.FindStringSubmatch(strings.TrimSpace(s)) + if m == nil { + return "", "", 0, fmt.Errorf("%q is not a pull request: name it owner/repository#number", s) + } + n, _ := strconv.Atoi(m[3]) + if n <= 0 { + return "", "", 0, fmt.Errorf("%q is not a pull request: its number is above zero", s) + } + return m[1], m[2], n, nil +} + +// pullKey is a pull request's key in the states. +func pullKey(pull string) string { + return strings.NewReplacer("/", ".", "#", ".").Replace(strings.TrimSpace(pull)) +} + +// PullDecl is what the controller told: the config declarations a pull request's head adds or changes, by module. +type PullDecl struct { + Pull string `json:"pull"` + Head string `json:"head"` + Declares map[string]map[string]Declaration `json:"declares"` + At time.Time `json:"at"` +} + +// KeptValue is an approved held value, kept apart from the values mesh-config resolves. +type KeptValue struct { + Pull string `json:"pull"` + Head string `json:"head"` + Digest string `json:"declaration"` + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + By string `json:"by"` + Via string `json:"via"` + Why string `json:"why"` + // Approved is who approved it, as the proposal kept it. + Approved string `json:"approved"` + Proposal string `json:"proposal"` + At time.Time `json:"at"` +} + +func keptKey(pull, node, module, key string) string { + return pullKey(pull) + "." + node + "." + module + "." + key +} + +func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } + +func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } + +// declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same. +func declDigest(d Declaration) string { + d.Description, d.Why, d.Kind, d.Runtime = "", "", "", false + if !d.HasDefault() { + d.Default = nil + } else { + d.Default = compact(d.Default) + } + raw, _ := json.Marshal(d) + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// PullForge is the forge as this module asks it of a pull request: its state, and whether it merged. +type PullForge interface { + Pull(owner, repo string, number int) (open, merged bool, err error) +} + +// pulls reads what is kept for pull requests, and the held values. +func (c *Config) readPulls(s *Sources) error { + s.Pulls, s.Kept = map[string]PullDecl{}, map[string]KeptValue{} + if err := c.each(PullsState, func(key string, raw []byte) error { + var p PullDecl + if err := json.Unmarshal(raw, &p); err != nil || p.Pull == "" { + return nil // not one this module wrote + } + s.Pulls[pullKey(p.Pull)] = p + return nil + }); err != nil { + return fmt.Errorf("the pull requests' declarations cannot be read: %w", err) + } + if err := c.each(KeptState, func(key string, raw []byte) error { + var k KeptValue + if err := json.Unmarshal(raw, &k); err != nil || k.Pull == "" { + return nil + } + s.Kept[key] = k + return nil + }); err != nil { + return fmt.Errorf("the held values cannot be read: %w", err) + } + return nil +} + +// ---- pull-declares --------------------------------------------------------------------------------------- + +// PullDeclaresAnswer is what pull-declares answers. +type PullDeclaresAnswer struct { + Said string `json:"said"` + Pull string `json:"pull"` + Head string `json:"head"` + Modules []string `json:"modules"` + Dropped []string `json:"dropped,omitempty"` + Unsaid []string `json:"unsaid,omitempty"` +} + +// PullDeclares keeps what a pull request's head declares, from the controller alone, and judges again the values +// held for it: a value whose key is declared the same (by digest) and still passes is kept at the new head; any +// other is dropped, recorded and raised, and must be proposed again (ADR 0502 clause 6). +func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { + if call.Caller != ControllerCaller { + return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+ + "mesh-config what the head declares; %s is not the controller", call.Caller) + } + if _, _, _, err := parsePull(call.Pull); err != nil { + return nil, err + } + head := strings.TrimSpace(call.Head) + if head == "" { + return nil, errors.New("name the head the declarations are read at") + } + raw, err := json.Marshal(call.Declares) + if err != nil { + return nil, fmt.Errorf("the declarations cannot be read: %v", err) + } + declares := map[string]map[string]Declaration{} + if call.Declares != nil { + if err := json.Unmarshal(raw, &declares); err != nil { + return nil, fmt.Errorf("declares is the modules' declarations, a key to a declaration for each module: %v", err) + } + } + for m, keys := range declares { + if !nodeOrModule.MatchString(m) { + return nil, fmt.Errorf("%q is not a module's name", m) + } + for k, d := range keys { + if !keyName.MatchString(k) { + return nil, fmt.Errorf("%q is not a config key", k) + } + if !d.HasDefault() { + d.Default = nil + } + keys[k] = d + } + } + pull := strings.TrimSpace(call.Pull) + c.mu.Lock() + defer c.mu.Unlock() + if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares, + At: c.now()}); err != nil { + return nil, fmt.Errorf("the declarations were not kept: %v", err) + } + a := &PullDeclaresAnswer{Pull: pull, Head: head, Modules: sortedKeys(declares)} + var s Sources + if err := c.readPulls(&s); err != nil { + return nil, err + } + for _, k := range sortedKeys(s.Kept) { + kv := s.Kept[k] + if pullKey(kv.Pull) != pullKey(pull) { + continue + } + why := "" + d, declared := declares[kv.Module][kv.Key] + switch { + case !declared: + why = "the head no longer adds or changes its declaration" + case declDigest(d) != kv.Digest: + why = "the head declares the key otherwise than the value was approved against" + default: + var v any + _ = json.Unmarshal(kv.Value, &v) + if p := d.rule().check(v); len(p) > 0 { + why = "the value no longer passes: " + strings.Join(p, "; ") + } + } + if why != "" { + a.Unsaid = append(a.Unsaid, c.dropKept(kv, "the new head "+short(head)+": "+why)...) + a.Dropped = append(a.Dropped, fmt.Sprintf("%s for %s on %s: %s", kv.Key, kv.Module, kv.Node, why)) + continue + } + if kv.Head != head { + kv.Head = head + if err := c.Store.Put(KeptState, k, kv); err != nil { + a.Unsaid = append(a.Unsaid, "the held value "+k+" was not moved to the new head: "+err.Error()) + } + } + } + a.Said = fmt.Sprintf("keeps what %s declares at %s for %d module(s)", pull, short(head), len(declares)) + if len(a.Dropped) > 0 { + a.Said += fmt.Sprintf("; %d held value(s) dropped", len(a.Dropped)) + } + return a, nil +} + +// heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and +// config-changed; what could not be done is answered, not dropped. +func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string { + level := kv.level() + ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why, + Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}} + unsaid := c.record(level.ValueKey(kv.Key), ch) + cause := "a value held for " + kv.Pull + what := fmt.Sprintf("config %s %s of %s for %s, held for %s", action, kv.Key, kv.Module, kv.Node, kv.Pull) + if action != ActionDrop { + what += " (" + string(kv.Value) + ")" + } + if err := c.Store.HandAct(what, why, cause); err != nil { + unsaid = append(unsaid, "the hand-act log does not hold it: "+err.Error()) + } + return unsaid +} + +// dropKept removes a held value, saying why. +func (c *Config) dropKept(kv KeptValue, why string) []string { + if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil { + return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()} + } + c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why) + return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "") +} + +// ---- held ------------------------------------------------------------------------------------------------ + +// HeldAnswer is what held answers: the approved held values of the pull requests asked, at the heads asked. +type HeldAnswer struct { + Held []HeldValue `json:"held"` + Skipped []HeldSkipped `json:"skipped"` +} + +// HeldValue is one approved held value as it is handed to a composed check and listed. +type HeldValue struct { + Pull string `json:"pull"` + Head string `json:"head"` + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Approved string `json:"approved"` + By string `json:"by,omitempty"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` +} + +// HeldSkipped is a pull request asked for whose held values are not answered, and why. +type HeldSkipped struct { + Pull string `json:"pull"` + Why string `json:"why"` +} + +func heldValueOf(k KeptValue) HeldValue { + return HeldValue{Pull: k.Pull, Head: k.Head, Node: k.Node, Module: k.Module, Key: k.Key, Value: compact(k.Value), + Approved: k.Approved, By: k.By, Why: k.Why, At: k.At} +} + +// Held answers the approved held values for the pull requests asked, at the heads asked, to the controller alone: a +// pull request at another head than the one kept is skipped and said (ADR 0502 clause 4). +func (c *Config) Held(call Call) (*HeldAnswer, error) { + if call.Caller != ControllerCaller { + return nil, fmt.Errorf("held is the controller's: it hands the held values to a composed check; %s is not "+ + "the controller", call.Caller) + } + asked := map[string]string{} + var order []string + for _, p := range call.Pulls { + if _, _, _, err := parsePull(p.Pull); err != nil { + return nil, err + } + if strings.TrimSpace(p.Head) == "" { + return nil, fmt.Errorf("name the head of %s", p.Pull) + } + if _, dup := asked[pullKey(p.Pull)]; !dup { + order = append(order, strings.TrimSpace(p.Pull)) + } + asked[pullKey(p.Pull)] = strings.TrimSpace(p.Head) + } + c.mu.Lock() + defer c.mu.Unlock() + var s Sources + if err := c.readPulls(&s); err != nil { + return nil, err + } + out := &HeldAnswer{Held: []HeldValue{}, Skipped: []HeldSkipped{}} + for _, p := range order { + head := asked[pullKey(p)] + var mine []KeptValue + for _, k := range sortedKeys(s.Kept) { + if pullKey(s.Kept[k].Pull) == pullKey(p) { + mine = append(mine, s.Kept[k]) + } + } + for _, k := range mine { + if k.Head != head { + out.Skipped = append(out.Skipped, HeldSkipped{Pull: p, Why: fmt.Sprintf("%s for %s on %s is held "+ + "at head %s, not %s", k.Key, k.Module, k.Node, short(k.Head), short(head))}) + continue + } + out.Held = append(out.Held, heldValueOf(k)) + } + } + return out, nil +} + +// HeldListed is the held values a listing shows, narrowed by module, node and key. +func heldListed(s Sources, call Call) []HeldValue { + out := []HeldValue{} + for _, k := range sortedKeys(s.Kept) { + kv := s.Kept[k] + if (call.Module == "" || kv.Module == call.Module) && (call.Node == "" || kv.Node == call.Node) && + (call.Key == "" || kv.Key == call.Key) { + out = append(out, heldValueOf(kv)) + } + } + return out +} + +// ---- the proposal's rule ------------------------------------------------------------------------------- + +// heldRule is the rule a value held for a pull request is judged by: the pull request's declaration of the key for +// the module at its head, and the module's assignment to the node. Refused, saying what the pull request does +// declare, when it declares none. +func heldRule(s Sources, call Call, level Level) (rule, PullDecl, string, error) { + if level.Kind != LevelModule { + return rule{}, PullDecl{}, "", fmt.Errorf("a value held for a pull request is a module's own value on one "+ + "node, and %s is not one: a predefined key a pull request adds is set after its merge (novox/hq ADR 0502)", + level) + } + pd, ok := s.Pulls[pullKey(call.Pull)] + if !ok { + return rule{}, PullDecl{}, "", fmt.Errorf("mesh-config holds no declaration for %s: the controller tells it "+ + "what a pull request declares when its head is announced, and none has been", strings.TrimSpace(call.Pull)) + } + d, ok := pd.Declares[level.Module][call.Key] + if !ok { + return rule{}, PullDecl{}, "", fmt.Errorf("%s does not add or change a value %q for %s at %s; it declares "+ + "for %s: %s", pd.Pull, call.Key, level.Module, short(pd.Head), level.Module, joinKeys(pd.Declares[level.Module])) + } + if _, reg := s.Declared[level.Module]; !reg { + return rule{}, PullDecl{}, "", fmt.Errorf("%s is not a registered module, so a value cannot be held for it", level.Module) + } + if !assigned(s, level.Node, level.Module) { + return rule{}, PullDecl{}, "", fmt.Errorf("%s is not assigned to %s, and a module's own value is set for a node "+ + "it is assigned to (novox/hq ADR 0348 §3); it is assigned to: %s", level.Module, level.Node, + strings.Join(nodesOf(s, level.Module), ", ")) + } + return d.rule(), pd, declDigest(d), nil +} + +// keep holds an approved proposal's value apart. The caller holds the lock. +func (c *Config) keep(p Proposal) (string, error) { + var s Sources + if err := c.readPulls(&s); err != nil { + return "", err + } + pd, ok := s.Pulls[pullKey(p.Pull)] + if !ok { + return "", fmt.Errorf("mesh-config holds no declaration for %s any more", p.Pull) + } + d, ok := pd.Declares[p.Level.Module][p.Key] + if !ok || declDigest(d) != p.DeclDigest { + return "", fmt.Errorf("%s declares %s otherwise than the operator was shown, or no longer declares it: "+ + "propose it again", p.Pull, p.Key) + } + was, had := s.Kept[keptKey(p.Pull, p.Level.Node, p.Level.Module, p.Key)] + if had != p.HadValue || valueDigest(was.Value, had) != p.WasDigest { + return "", fmt.Errorf("the value held for %s changed since the operator was shown the change: propose it "+ + "again", p.Pull) + } + kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module, + Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID, + At: c.now()} + if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil { + return "", fmt.Errorf("the value was not kept: %v", err) + } + unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved) + said := fmt.Sprintf("%s of %s on %s is held at %s for %s until it merges", p.Key, p.Level.Module, p.Level.Node, + short(pd.Head), p.Pull) + if len(unsaid) > 0 { + said += " (" + strings.Join(unsaid, "; ") + ")" + } + return said, nil +} + +// ---- the merge and the close --------------------------------------------------------------------------- + +// PullMerged writes each value held for a pull request as the module's own value on its node, with the approval kept +// beside it; it resolves once the trunk declares the key. It answers a line for each. +func (c *Config) PullMerged(pull string) []string { + return c.settlePull(pull, true) +} + +// PullClosed drops the values held for a pull request closed unmerged, and says that nothing was set. +func (c *Config) PullClosed(pull string) []string { + return c.settlePull(pull, false) +} + +func (c *Config) settlePull(pull string, merged bool) []string { + c.mu.Lock() + defer c.mu.Unlock() + var s Sources + if err := c.readPulls(&s); err != nil { + return []string{err.Error()} + } + var said []string + wrote := false + for _, k := range sortedKeys(s.Kept) { + kv := s.Kept[k] + if pullKey(kv.Pull) != pullKey(pull) { + continue + } + if !merged { + said = append(said, fmt.Sprintf("%s of %s on %s was held for %s, which closed unmerged: dropped, "+ + "nothing was set", kv.Key, kv.Module, kv.Node, kv.Pull)) + said = append(said, c.dropKept(kv, kv.Pull+" closed unmerged")...) + continue + } + vk := kv.level().ValueKey(kv.Key) + v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull + + ", merged)", Approved: kv.Approved, At: c.now()} + if err := c.Store.Put(ValuesState, vk, v); err != nil { + said = append(said, fmt.Sprintf("%s of %s on %s held for %s is not written yet: %v; it is tried "+ + "again", kv.Key, kv.Module, kv.Node, kv.Pull, err)) + continue + } + if err := c.Store.Delete(KeptState, k); err != nil { + said = append(said, "the held value "+k+" is written but not removed: "+err.Error()) + } + wrote = true + said = append(said, c.heldChange(kv, "set", kv.Pull+" merged: the held value is the module's own value now", + kv.Approved)...) + said = append(said, fmt.Sprintf("%s of %s on %s held for %s is set: %s", kv.Key, kv.Module, kv.Node, + kv.Pull, kv.Value)) + } + if wrote { + if s2, err := c.sources(); err == nil { + if _, err := c.writeResolved(s2); err != nil { + said = append(said, "the resolved values are not all written yet ("+err.Error()+"); the next "+ + "resolution writes them") + } + } + } + left := 0 + for _, k := range sortedKeys(s.Kept) { + if pullKey(s.Kept[k].Pull) != pullKey(pull) { + continue + } + if e, err := c.Store.Get(KeptState, k); err != nil || e != nil { + left++ + } + } + if left == 0 { + if err := c.Store.Delete(PullsState, pullKey(pull)); err != nil { + said = append(said, "the declarations of "+pull+" were not removed: "+err.Error()) + } + } + return said +} + +// PullEvent is the forge's `pull.merged` or `pull.closed`, as far as this module reads it. +type PullEvent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` + Merged bool `json:"merged,omitempty"` +} + +func (p PullEvent) name() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) } + +// HeardPull takes the forge's word that a pull request merged or closed. A close is asked of the forge when it can +// be, because a merged pull request may be announced as closed too. +func (c *Config) HeardPull(p PullEvent, merged bool) []string { + if p.Owner == "" || p.Repo == "" || p.Number <= 0 { + return nil + } + if !merged && !p.Merged && c.Forge != nil { + if open, m, err := c.Forge.Pull(p.Owner, p.Repo, p.Number); err == nil { + if open { + return nil + } + merged = m + } + } + return c.settlePull(p.name(), merged || p.Merged) +} + +// SweepPulls looks at the forge for every pull request it keeps something for, so a merge or a close it did not hear +// is found: run with each five-minute resolution. +func (c *Config) SweepPulls() []string { + if c.Forge == nil { + return nil + } + c.mu.Lock() + var s Sources + err := c.readPulls(&s) + c.mu.Unlock() + if err != nil { + return []string{err.Error()} + } + seen := map[string]string{} + for _, p := range s.Pulls { + seen[pullKey(p.Pull)] = p.Pull + } + for _, k := range s.Kept { + seen[pullKey(k.Pull)] = k.Pull + } + var said []string + for _, key := range sortedKeys(seen) { + pull := seen[key] + owner, repo, n, err := parsePull(pull) + if err != nil { + continue + } + open, merged, err := c.Forge.Pull(owner, repo, n) + if err != nil { + said = append(said, fmt.Sprintf("the forge does not say whether %s is open: %v", pull, err)) + continue + } + if !open { + said = append(said, c.settlePull(pull, merged)...) + } + } + return said +} + +// PullAt is a pull request at a head, as held asks about it. +type PullAt struct { + Pull string `json:"pull"` + Head string `json:"head"` +} + +// toolForge asks the forge's holder through its tools. +type toolForge struct { + ask func(key string, body any) (json.RawMessage, error) +} + +// Pull asks the forge's gitea_get_pull_request whether a pull request is open, and whether it merged. +func (f toolForge) Pull(owner, repo string, number int) (bool, bool, error) { + raw, err := f.ask("gitea.gitea_get_pull_request", map[string]any{"owner": owner, "repo": repo, "number": number}) + if err != nil { + return false, false, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + return false, false, fmt.Errorf("the forge refused gitea_get_pull_request: %s", output) + } + var a struct { + Pull struct { + State string `json:"state"` + Merged bool `json:"merged"` + } `json:"pull"` + } + if err := json.Unmarshal(answer, &a); err != nil || a.Pull.State == "" { + return false, false, fmt.Errorf("the forge's gitea_get_pull_request is not readable: %s", output) + } + return a.Pull.State == "open", a.Pull.Merged, nil +} + +// unwrapAnswer reads a tool's answer whatever wraps it: the protocol's content list, or {output, ok, answer}. +func unwrapAnswer(raw json.RawMessage) (json.RawMessage, string, bool) { + var s string + if json.Unmarshal(raw, &s) == nil { + return unwrapAnswer(json.RawMessage(s)) + } + var m map[string]json.RawMessage + if json.Unmarshal(raw, &m) != nil { + return raw, string(raw), true + } + if content, has := m["content"]; has { + var items []struct { + Text string `json:"text"` + } + var e struct { + IsError bool `json:"isError"` + } + _ = json.Unmarshal(raw, &e) + if json.Unmarshal(content, &items) == nil && len(items) > 0 { + inner, out, ok := unwrapAnswer(json.RawMessage(items[0].Text)) + return inner, out, ok && !e.IsError + } + } + var v struct { + Output string `json:"output"` + OK *bool `json:"ok"` + Answer json.RawMessage `json:"answer"` + } + if json.Unmarshal(raw, &v) == nil && v.OK != nil { + return v.Answer, v.Output, *v.OK + } + return raw, string(raw), true +} diff --git a/modules/mesh-config/cmd/mesh-config/pulls_test.go b/modules/mesh-config/cmd/mesh-config/pulls_test.go new file mode 100644 index 00000000..e8a514b6 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/pulls_test.go @@ -0,0 +1,445 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +// Issue 1826 (novox/hq ADR 0502): a value held for the open pull request that introduces its key. + +const thePull = "novox/mesh-catalog#434" + +// n8nPullDeclares is what the controller tells of the pull request: n8n gets a required number media-gid. +func n8nPullDeclares() map[string]any { + return map[string]any{"n8n": map[string]any{"media-gid": map[string]any{"type": "number", + "description": "the media group", "required": true, "minimum": 1, "maximum": 65535}}} +} + +// aHoldingMesh is a proposing mesh with n8n assigned to ace and declaring nothing on the trunk (the case of the +// issue), and a stand-in forge. +func aHoldingMesh(t *testing.T) (*Config, *memStore, *standIn, *standForge) { + t.Helper() + c, m, r := aProposingMesh(t) + m.mirror(t, DeclaredConfigRead, "n8n", `{"module":"n8n","settings":{}}`) + m.assign(t, "ace", "n8n") + f := &standForge{state: map[string][2]bool{}} + c.Forge = f + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + m.events, m.handActs = nil, nil + return c, m, r, f +} + +type standForge struct { + state map[string][2]bool // pull -> open, merged + err error +} + +func (f *standForge) Pull(owner, repo string, n int) (bool, bool, error) { + if f.err != nil { + return false, false, f.err + } + st, ok := f.state[(PullEvent{Owner: owner, Repo: repo, Number: n}).name()] + if !ok { + return true, false, nil + } + return st[0], st[1], nil +} + +func tell(t *testing.T, c *Config, pull, head string, declares map[string]any) *PullDeclaresAnswer { + t.Helper() + a, err := c.Do(Call{Action: "pull-declares", Pull: pull, Head: head, Declares: declares, Caller: ControllerCaller}) + if err != nil { + t.Fatalf("pull-declares: %v", err) + } + return a.(*PullDeclaresAnswer) +} + +// holdIt proposes media-gid=2000 for n8n on ace held for the pull request and has the operator approve it. +func holdIt(t *testing.T, c *Config, m *memStore, r *standIn) string { + t.Helper() + a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, + Why: "n8n's container joins the media group, held for #434"}) + r.choose(t, a.Proposal, OptionApprove, nil) + said, err := c.Decide() + if err != nil || len(said) != 1 || !strings.Contains(said[0], "approved by the operator and held") { + t.Fatalf("the Approve decided %v (%v)", said, err) + } + return a.Proposal +} + +func keptCount(m *memStore) int { k, _ := m.Keys(KeptState); return len(k) } + +// 1826/K1: mesh-config takes what a pull request declares from the controller alone, and keeps it per pull request. +func TestAPullRequestsDeclarationsAreTakenFromTheControllerAlone(t *testing.T) { + c, m, _, _ := aHoldingMesh(t) + for _, caller := range []string{"g14.node-tools", Unproven, ""} { + if _, err := c.Do(Call{Action: "pull-declares", Pull: thePull, Head: "abc1234", Declares: n8nPullDeclares(), + Caller: caller}); err == nil || !strings.Contains(err.Error(), "controller") { + t.Errorf("pull-declares from %q: %v", caller, err) + } + } + for _, bad := range []Call{{Pull: "mesh-catalog#434", Head: "a"}, {Pull: thePull}, {Pull: "a/b#0", Head: "a"}} { + bad.Action, bad.Caller = "pull-declares", ControllerCaller + if _, err := c.Do(bad); err == nil { + t.Errorf("pull-declares %+v was taken", bad) + } + } + a := tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + if len(a.Modules) != 1 || a.Modules[0] != "n8n" { + t.Errorf("answered %+v", a) + } + e, _ := m.Get(PullsState, "novox.mesh-catalog.434") + var pd PullDecl + if e == nil || json.Unmarshal(e.Value, &pd) != nil || pd.Head != "abc1234def" || + pd.Declares["n8n"]["media-gid"].Type != "number" || !pd.Declares["n8n"]["media-gid"].Required { + t.Errorf("kept %s", func() string { + if e == nil { + return "nothing" + } + return string(e.Value) + }()) + } + // Nothing is resolved from a pull request's declaration. + if r, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); r != nil { + t.Errorf("a pull request's declaration was resolved: %s", r.Value) + } +} + +// 1826/K2: a proposal for a key only the pull request declares is refused without its declaration, for a mesh or +// node level, for a clear, for an undeclared key, and for a module not assigned to the node; and trunk-only +// proposals are judged as before. +func TestAProposalNamingAPullRequestIsJudgedAgainstItsDeclaration(t *testing.T) { + c, m, _, _ := aHoldingMesh(t) + try := func(call Call) error { + call.Action, call.Caller, call.Why = "propose", "g14.node-tools", "held for 434" + _, err := c.Propose(call) + return err + } + base := Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull} + if err := try(base); err == nil || !strings.Contains(err.Error(), "holds no declaration for "+thePull) { + t.Errorf("without the pull request's declaration: %v", err) + } + // And without the pull argument it is the trunk's refusal, as before. + plain := base + plain.Pull = "" + if err := try(plain); err == nil || !strings.Contains(err.Error(), `n8n declares no value "media-gid"`) { + t.Errorf("the trunk's refusal changed: %v", err) + } + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + for name, tc := range map[string]struct { + mutate func(*Call) + want string + }{ + "a mesh level": {func(c *Call) { c.Module, c.Node = "", "" }, "a module's own value on one node"}, + "a node level": {func(c *Call) { c.Module = "" }, "a module's own value on one node"}, + "a clear": {func(c *Call) { c.Clear, c.Value = true, nil }, "a clear is not held"}, + "a key not added": {func(c *Call) { c.Key = "other" }, "does not add or change a value"}, + "a bad value": {func(c *Call) { c.Value = 0.0 }, "below the minimum"}, + "a wrong type": {func(c *Call) { c.Value = "2000" }, "is not a number"}, + "another node": {func(c *Call) { c.Node = "g14" }, "is not assigned to g14"}, + "another pull": {func(c *Call) { c.Pull = "novox/mesh-catalog#435" }, "holds no declaration for"}, + "a malformed pull": {func(c *Call) { c.Pull = "434" }, "owner/repository#number"}, + } { + call := base + tc.mutate(&call) + if err := try(call); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: %v, want %q", name, err, tc.want) + } + } + // A set or a clear through the verb takes no pull request. + if _, err := c.Change(Call{Action: "set", Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, + Caller: "g14.node-tools", Why: "w"}); err == nil || !strings.Contains(err.Error(), "only a proposal") { + t.Errorf("a set naming a pull request: %v", err) + } + if keptCount(m) != 0 { + t.Error("something was held") + } +} + +// 1826/K2: every held value is asked whatever the key's trust; the ask says it is held and dropped on a close; on +// Approve the value is kept apart, never set or resolved; history, event and hand-act say it; held answers it. +func TestAHeldValueIsAskedAndKeptApartOnApprove(t *testing.T) { + c, m, r, _ := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, Why: "held for #434"}) + if len(a.Reaches) != 0 { + t.Errorf("a held value reaches %v", a.Reaches) + } + ask := r.asks[a.Proposal] + if err := ask.Check(c.now()); err != nil { + t.Fatalf("the ask: %v", err) + } + for _, want := range []string{thePull, "merges", "dropped if it closes unmerged", "media-gid", "n8n", "ace"} { + if !strings.Contains(ask.Explanation, want) { + t.Errorf("the ask does not say %q:\n%s", want, ask.Explanation) + } + } + if strings.Contains(ask.Explanation, "takes effect at once") || !strings.Contains(ask.Headline, "Hold") { + t.Errorf("the ask: %q\n%s", ask.Headline, ask.Explanation) + } + if ap := ask.Options[0]; !strings.Contains(ap.Does, thePull) || ap.Level == "" { + t.Errorf("the approve option does %q", ap.Does) + } + if keptCount(m) != 0 { + t.Fatal("a proposal held something") + } + r.choose(t, a.Proposal, OptionApprove, nil) + said, _ := c.Decide() + if len(said) != 1 || !strings.Contains(said[0], "held at abc1234d for "+thePull) { + t.Fatalf("decided %v", said) + } + // Kept apart: not a value, not resolved. + if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set { + t.Error("an Approve set the value") + } + if e, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); e != nil { + t.Errorf("an Approve resolved the value: %s", e.Value) + } + if keptCount(m) != 1 { + t.Fatalf("%d values held", keptCount(m)) + } + if len(m.events) != 1 || m.events[0].Action != ActionHold || m.events[0].Pull != thePull || + m.events[0].Key != "media-gid" || m.events[0].Head != "abc1234def" || len(m.events[0].Moved) != 0 || + !strings.Contains(m.events[0].Approved, "approved by the operator") { + t.Errorf("raised %+v", m.events) + } + if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], thePull) { + t.Errorf("hand-acts %q", m.handActs) + } + h, _ := c.History(Call{Module: "n8n", Node: "ace"}) + if len(h) != 1 || h[0].Action != ActionHold || h[0].Pull != thePull { + t.Errorf("history %+v", h) + } + // held answers it, at the head it is kept at, to the controller alone. + if _, err := c.Do(Call{Action: "held", Caller: "g14.node-tools", Pulls: []PullAt{{thePull, "abc1234def"}}}); err == nil { + t.Error("held answered a caller that is not the controller") + } + got, err := c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "abc1234def"}}}) + if err != nil { + t.Fatal(err) + } + ha := got.(*HeldAnswer) + if len(ha.Held) != 1 || ha.Held[0].Module != "n8n" || ha.Held[0].Node != "ace" || ha.Held[0].Key != "media-gid" || + string(ha.Held[0].Value) != "2000" || !strings.Contains(ha.Held[0].Approved, "approved by the operator") || + ha.Held[0].Pull != thePull || len(ha.Skipped) != 0 { + t.Errorf("held answered %+v", ha) + } + // Another head, and another pull request, are not answered. + got, _ = c.Do(Call{Action: "held", Caller: ControllerCaller, + Pulls: []PullAt{{thePull, "ffff0000"}, {"novox/mesh-catalog#435", "abc"}}}) + if ha = got.(*HeldAnswer); len(ha.Held) != 0 || len(ha.Skipped) != 1 || !strings.Contains(ha.Skipped[0].Why, "held at") { + t.Errorf("held at another head answered %+v", ha) + } + // The listing shows it as held, and the proposal as held for the pull request. + l, _ := c.List(Call{Module: "n8n"}) + if len(l.Held) != 1 || len(l.Set) != 0 { + t.Errorf("the listing: held %+v set %+v", l.Held, l.Set) + } + rows, _ := c.Proposals(Call{}) + if len(rows) != 1 || rows[0].Pull != thePull || rows[0].State != ProposalApproved { + t.Errorf("proposals %+v", rows) + } +} + +// 1826/K2: an Approve is refused when the pull request's declaration changed after the operator was shown it. +func TestAnApproveOfAHeldValueIsRefusedWhenTheDeclarationChanged(t *testing.T) { + c, m, r, _ := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, Why: "w"}) + changed := n8nPullDeclares() + changed["n8n"].(map[string]any)["media-gid"].(map[string]any)["maximum"] = 1000.0 + tell(t, c, thePull, "def5678abc", changed) + r.choose(t, a.Proposal, OptionApprove, nil) + said, _ := c.Decide() + if len(said) != 1 || !strings.Contains(said[0], "nothing is held") || keptCount(m) != 0 { + t.Errorf("decided %v; %d held", said, keptCount(m)) + } +} + +// 1826/K1: at a new head a held value is kept if its key is declared the same and it still passes; otherwise it is +// dropped, recorded, raised and said. +func TestANewHeadKeepsAHeldValueOnlyWhileItsDeclarationIsUnchanged(t *testing.T) { + c, m, r, _ := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + holdIt(t, c, m, r) + m.events = nil + + // The same declaration at a new head (the description may differ): kept, moved to the new head. + same := n8nPullDeclares() + same["n8n"].(map[string]any)["media-gid"].(map[string]any)["description"] = "reworded" + if a := tell(t, c, thePull, "1111111111", same); len(a.Dropped) != 0 { + t.Errorf("an unchanged declaration dropped %v", a.Dropped) + } + got, _ := c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "1111111111"}}}) + if len(got.(*HeldAnswer).Held) != 1 || len(m.events) != 0 { + t.Errorf("after an unchanged head: %+v, events %+v", got, m.events) + } + + // A changed declaration: dropped. + changed := n8nPullDeclares() + changed["n8n"].(map[string]any)["media-gid"].(map[string]any)["maximum"] = 1000.0 + a := tell(t, c, thePull, "2222222222", changed) + if len(a.Dropped) != 1 || !strings.Contains(a.Dropped[0], "declares the key otherwise") || keptCount(m) != 0 { + t.Fatalf("a changed declaration: %+v, %d held", a, keptCount(m)) + } + if len(m.events) != 1 || m.events[0].Action != ActionDrop || m.events[0].Pull != thePull { + t.Errorf("raised %+v", m.events) + } + got, _ = c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "2222222222"}}}) + if len(got.(*HeldAnswer).Held) != 0 { + t.Error("a dropped value was answered") + } + + // A head that no longer adds the key drops it too. + tell(t, c, thePull, "3333333333", n8nPullDeclares()) + holdIt(t, c, m, r) + if a = tell(t, c, thePull, "4444444444", map[string]any{}); len(a.Dropped) != 1 || + !strings.Contains(a.Dropped[0], "no longer adds or changes") { + t.Errorf("a head without the key: %+v", a) + } +} + +// 1826/K3: the merge writes the held value as the module's own value with its approval kept, before the trunk +// declares the key; it resolves right after the declared-config mirror moves. +func TestTheMergeAppliesAHeldValueAndItResolvesOnceTheTrunkDeclaresTheKey(t *testing.T) { + c, m, r, _ := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + holdIt(t, c, m, r) + approved := func() string { + raw, _ := m.Get(KeptState, "novox.mesh-catalog.434.ace.n8n.media-gid") + var k KeptValue + _ = json.Unmarshal(raw.Value, &k) + return k.Approved + }() + m.events, m.handActs = nil, nil + + said := c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true}, true) + if len(said) == 0 { + t.Fatal("the merge said nothing") + } + v, set := storedValue(t, m, "module.ace.n8n.media-gid") + if !set || string(v.Value) != "2000" || v.Approved != approved || v.Approved == "" || v.By != "the operator" || + !strings.Contains(v.Why, thePull) { + t.Fatalf("the merge wrote %+v", v) + } + if keptCount(m) != 0 { + t.Error("the value is still held after the merge") + } + if p, _ := m.Keys(PullsState); len(p) != 0 { + t.Errorf("the pull request's declarations are still kept: %v", p) + } + if len(m.events) != 1 || m.events[0].Action != "set" || m.events[0].Pull != thePull || + m.events[0].Approved != approved || len(m.handActs) != 1 { + t.Errorf("raised %+v, recorded %q", m.events, m.handActs) + } + // Not resolved before the trunk declares it. + if e, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); e != nil { + t.Errorf("resolved before the trunk declares the key: %s", e.Value) + } + m.mirror(t, DeclaredConfigRead, "n8n", `{"module":"n8n","settings":{"media-gid":{"kind":"","default":null,"why":"",`+ + `"type":"number","required":true,"minimum":1,"maximum":65535}}}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + if res := m.resolved(t, "ace", "n8n", "media-gid"); string(res.Value) != "2000" || res.Source != SourceModule { + t.Errorf("media-gid resolves as %+v", res) + } + l, _ := c.List(Call{Module: "n8n", Node: "ace"}) + if len(l.Set) != 1 || l.Set[0].Approved != approved || len(l.Held) != 0 { + t.Errorf("the listing: set %+v held %+v", l.Set, l.Held) + } + // A merge heard twice sets nothing twice. + m.events = nil + c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true}, true) + if len(m.events) != 0 { + t.Errorf("a second merge raised %+v", m.events) + } +} + +// 1826/K3: a pull request closed unmerged drops its held values and says so; nothing is set or resolved. +func TestAClosedPullRequestDropsItsHeldValuesAndSaysSo(t *testing.T) { + c, m, r, f := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + holdIt(t, c, m, r) + m.events = nil + f.state[thePull] = [2]bool{false, false} + said := c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434}, false) + if len(said) == 0 || !strings.Contains(strings.Join(said, "\n"), "closed unmerged: dropped, nothing was set") { + t.Errorf("said %v", said) + } + if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set || keptCount(m) != 0 { + t.Error("a closed pull request left a value") + } + if len(m.events) != 1 || m.events[0].Action != ActionDrop || m.events[0].Pull != thePull || + !strings.Contains(m.events[0].Why, "closed unmerged") { + t.Errorf("raised %+v", m.events) + } + if p, _ := m.Keys(PullsState); len(p) != 0 { + t.Errorf("declarations of a closed pull request kept: %v", p) + } +} + +// 1826/K3: a close event for a pull request the forge says merged is the merge. +func TestACloseEventForAMergedPullRequestIsItsMerge(t *testing.T) { + c, m, r, f := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + holdIt(t, c, m, r) + f.state[thePull] = [2]bool{false, true} + c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434}, false) + if v, set := storedValue(t, m, "module.ace.n8n.media-gid"); !set || string(v.Value) != "2000" { + t.Errorf("the merge of a pull request announced closed set %+v", v) + } +} + +// 1826/K3: the five-minute look finds a merge, a close, and an open pull request, and says a forge that does not +// answer. +func TestTheLookAtTheForgeFindsAMissedMergeOrCloseAndLeavesAnOpenPullRequest(t *testing.T) { + c, m, r, f := aHoldingMesh(t) + tell(t, c, thePull, "abc1234def", n8nPullDeclares()) + holdIt(t, c, m, r) + if said := c.SweepPulls(); len(said) != 0 || keptCount(m) != 1 { + t.Errorf("an open pull request: said %v, %d held", said, keptCount(m)) + } + f.err = errors.New("the forge is down") + if said := c.SweepPulls(); len(said) != 1 || !strings.Contains(said[0], "does not say whether") || keptCount(m) != 1 { + t.Errorf("a silent forge: said %v", said) + } + f.err = nil + f.state[thePull] = [2]bool{false, true} + c.SweepPulls() + if v, set := storedValue(t, m, "module.ace.n8n.media-gid"); !set || string(v.Value) != "2000" || keptCount(m) != 0 { + t.Errorf("a missed merge: %+v", v) + } + // A missed close. + tell(t, c, "novox/mesh-catalog#440", "abc", n8nPullDeclares()) + f.state["novox/mesh-catalog#440"] = [2]bool{false, false} + c.SweepPulls() + if p, _ := m.Keys(PullsState); len(p) != 0 { + t.Errorf("a missed close left %v", p) + } +} + +// 1826/K1: the new arguments are read strictly and the new actions are the verb's. +func TestThePullArgumentsAreRead(t *testing.T) { + call, err := callOf(map[string]any{"action": "pull-declares", "pull": " " + thePull + " ", "head": "abc", + "declares": map[string]any{"n8n": map[string]any{}}, "pulls": []any{map[string]any{"pull": thePull, "head": "h"}}}) + if err != nil || call.Pull != thePull || call.Head != "abc" || len(call.Declares) != 1 || len(call.Pulls) != 1 || + call.Pulls[0].Head != "h" { + t.Errorf("read %+v (%v)", call, err) + } + if _, err := callOf(map[string]any{"declares": "x"}); err == nil { + t.Error("declares that is not an object was read") + } + if _, err := callOf(map[string]any{"pulls": "x"}); err == nil { + t.Error("pulls that is not a list was read") + } + if _, err := callOf(map[string]any{"bogus": 1}); err == nil || !strings.Contains(err.Error(), "pull") { + t.Errorf("an unknown argument: %v", err) + } + _ = time.Second +} diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go index f095f324..bc21ec61 100644 --- a/modules/mesh-config/cmd/mesh-config/resolve.go +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -127,6 +127,10 @@ type Sources struct { Declared map[string]ModuleDeclarations Predefined map[string]Published Values map[string]Value + // Pulls is what the controller told of each open pull request's declarations, and Kept the approved values held + // for them (pulls.go); neither is resolved. + Pulls map[string]PullDecl + Kept map[string]KeptValue } // Resolve is every assigned module's every declared value on its node, keyed as mesh-config_resolved is. diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index 221d1294..df6292dc 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -59,6 +59,9 @@ func (c *Config) sources() (Sources, error) { }); err != nil { return s, fmt.Errorf("the values cannot be read: %w", err) } + if err := c.readPulls(&s); err != nil { + return s, err + } for _, p := range problems { c.logf("[mesh-config] a module's declarations are not read, so its values are not resolved: %s", p) } diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index e03e891b..f93497e1 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -17,9 +17,15 @@ "uses": [ "operator-channel" ], + "consumes": [ + "gitea.pull.merged", + "gitea.pull.closed", + "gitea.forge.pull-request.closed" + ], "invokes": [ "seat:mesh-controller.hand-act", - "seat:mesh-controller.warranted" + "seat:mesh-controller.warranted", + "gitea.gitea_get_pull_request" ], "reads": [ "mesh-controller.assignments", @@ -30,7 +36,9 @@ "values", "resolved", "predefined", - "proposals" + "proposals", + "pulls", + "kept" ], "logs": [ "history"