Feature 57 (config and provisioning): modules declare config and provisions, their files rendered from templates; synced with main (issues of feature 57, 874)

This commit is contained in:
2026-10-16 07:37:43 +00:00
parent 1a2c917abf
commit 9652b2d5ce
771 changed files with 52063 additions and 1774 deletions
+42
View File
@@ -0,0 +1,42 @@
// agent-teams: the agents' read teams on the forge (novox/hq ADR 0365 clause 3, issue 758). A program the node's
// runtime runs once when the module starts and again whenever the module's config value `agent-read-teams` changes:
// it reads the file the mesh renders from that value, and makes the forge's teams named agent-read-* match it. Each
// team reads only; its repositories and its members (agent accounts, agent-<node>) are exactly the value's. It
// touches nothing outside the value: no team without the prefix, no repository or account of another kind.
//
// It says each change on stdout in one line, and never a secret.
package main
import (
"context"
"fmt"
"os"
"time"
"gitea/internal/forgehttp"
)
func main() {
forge, missing := forgehttp.FromEnv()
file := os.Getenv("MESH_AGENT_TEAMS_FILE")
if file == "" {
missing = append(missing, "MESH_AGENT_TEAMS_FILE")
}
if len(missing) > 0 {
fmt.Fprintf(os.Stderr, "agent-teams: %v not set by the manifest; nothing was changed\n", missing)
os.Exit(1)
}
value, err := ReadValue(file)
if err != nil {
fmt.Fprintln(os.Stderr, "agent-teams:", err)
os.Exit(1)
}
s := &Syncer{Forge: forge, Log: func(format string, a ...any) { fmt.Printf(format+"\n", a...) },
WaitFor: 3 * time.Minute, Poll: 5 * time.Second}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
if err := s.Sync(ctx, value); err != nil {
fmt.Fprintln(os.Stderr, "agent-teams:", err)
os.Exit(1)
}
}
+339
View File
@@ -0,0 +1,339 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"sort"
"strings"
"time"
"gitea/internal/forgehttp"
)
const (
// ConfigKey is the module's config value this program applies.
ConfigKey = "agent-read-teams"
// TeamPrefix names every team this program owns; a team without it is never touched (ADR 0365 §3).
TeamPrefix = "agent-read-"
// AccountPrefix names an agent's account for its node (cmd/agent-token AccountPrefix, ADR 0365 §1).
AccountPrefix = "agent-"
page = 50
)
var (
teamName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,40}$`)
nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`)
repoName = regexp.MustCompile(`^([A-Za-z0-9][A-Za-z0-9_.-]*)/([A-Za-z0-9_.-]+)$`)
)
// readUnits are the units a read team has, each at read: the forge's rights of a team that reads and nothing more.
var readUnits = map[string]string{
"repo.code": "read", "repo.issues": "read", "repo.pulls": "read", "repo.releases": "read", "repo.wiki": "read",
}
// Team is one read team of the value: its repositories (owner/repo, read only) and its members (nodes whose agent
// account, agent-<node>, is in the team).
type Team struct {
Repos []string `json:"repos"`
Members []string `json:"members"`
}
// Value is the config value: each team by the name it is given without the prefix.
type Value map[string]Team
// ReadValue reads the file the mesh renders from the config value. A file without the key is the empty value,
// which empties every read team.
func ReadValue(path string) (Value, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("the teams file cannot be read (%s): %w", path, err)
}
var file map[string]json.RawMessage
if err := json.Unmarshal(raw, &file); err != nil {
return nil, fmt.Errorf("the teams file %s is not JSON: %w", path, err)
}
var v Value
if r, ok := file[ConfigKey]; ok {
dec := json.NewDecoder(strings.NewReader(string(r)))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return nil, fmt.Errorf("%s in %s is not {team: {repos, members}}: %w", ConfigKey, path, err)
}
}
return v, nil
}
// Syncer makes the forge's agent-read-* teams the value's.
type Syncer struct {
Forge forgehttp.Client
Log func(format string, a ...any)
WaitFor time.Duration // how long to wait for the forge to answer; none when zero
Poll time.Duration
}
type forgeTeam struct {
ID int64 `json:"id"`
Name string `json:"name"`
Permission string `json:"permission"`
}
type forgeRepo struct {
FullName string `json:"full_name"`
}
type forgeMember struct {
Login string `json:"login"`
}
func (s *Syncer) say(format string, a ...any) {
if s.Log != nil {
s.Log(format, a...)
}
}
// Sync applies the value. A team that cannot be applied is said and the others still are; the error names them.
func (s *Syncer) Sync(ctx context.Context, v Value) error {
if err := s.waitForForge(ctx); err != nil {
return err
}
orgs, err := s.orgs(ctx)
if err != nil {
return err
}
wanted := map[string]bool{}
var failed []string
names := make([]string, 0, len(v))
for n := range v {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
wanted[TeamPrefix+n] = true
if err := s.apply(ctx, n, v[n]); err != nil {
s.say("not applied: team %s%s: %v", TeamPrefix, n, err)
failed = append(failed, TeamPrefix+n)
}
}
for _, org := range orgs {
teams, err := s.teams(ctx, org)
if err != nil {
return err
}
for _, t := range teams {
if !strings.HasPrefix(t.Name, TeamPrefix) || wanted[t.Name] {
continue
}
if err := s.sync(ctx, org, t, nil, nil); err != nil {
s.say("not applied: team %s in %s: %v", t.Name, org, err)
failed = append(failed, t.Name)
}
}
}
if len(failed) > 0 {
return fmt.Errorf("not applied: %s", strings.Join(failed, ", "))
}
return nil
}
func (s *Syncer) waitForForge(ctx context.Context) error {
deadline := time.Now().Add(s.WaitFor)
for {
err := s.Forge.GetJSON(ctx, "/api/v1/version", nil)
if err == nil {
return nil
}
var se *forgehttp.StatusError
if errors.As(err, &se) || time.Now().After(deadline) || ctx.Err() != nil {
return fmt.Errorf("the forge is not reachable: %w", err)
}
time.Sleep(s.Poll)
}
}
// apply makes one team of the value.
func (s *Syncer) apply(ctx context.Context, name string, t Team) error {
if !teamName.MatchString(name) {
return fmt.Errorf("%q is not a team name", name)
}
if len(t.Repos) == 0 {
return errors.New("it lists no repository, so the organisation to make it in is not known")
}
org := ""
repos := map[string]bool{}
for _, r := range t.Repos {
m := repoName.FindStringSubmatch(r)
if m == nil {
return fmt.Errorf("%q is not owner/repo", r)
}
if org == "" {
org = m[1]
}
if m[1] != org {
return fmt.Errorf("%q is not in %s: a team's repositories are of one owner, the first's", r, org)
}
repos[r] = true
}
members := map[string]bool{}
for _, n := range t.Members {
if !nodeName.MatchString(n) {
return fmt.Errorf("%q is not a node's name", n)
}
members[AccountPrefix+n] = true
}
full := TeamPrefix + name
teams, err := s.teams(ctx, org)
if err != nil {
return err
}
var team *forgeTeam
for i := range teams {
if teams[i].Name == full {
team = &teams[i]
}
}
if team == nil {
var made forgeTeam
err := s.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+org+"/teams", map[string]any{
"name": full, "description": "Read-only team of agents' forge accounts (ADR 0365 §3); set by the gitea config value " + ConfigKey,
"permission": "read", "includes_all_repositories": false, "can_create_org_repo": false, "units_map": readUnits,
}, &made)
if err != nil {
return fmt.Errorf("the team could not be made: %w", err)
}
s.say("made the team %s in %s, reading only", full, org)
team = &made
} else if team.Permission != "read" {
if err := s.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", team.ID), map[string]any{
"name": full, "permission": "read", "includes_all_repositories": false, "can_create_org_repo": false, "units_map": readUnits,
}, nil); err != nil {
return fmt.Errorf("the team could not be made read-only: %w", err)
}
s.say("made the team %s in %s read-only (it was %s)", full, org, team.Permission)
}
return s.sync(ctx, org, *team, repos, members)
}
// sync makes the team's repositories and members exactly the sets (nil is none).
func (s *Syncer) sync(ctx context.Context, org string, t forgeTeam, repos, members map[string]bool) error {
var errs []string
var haveRepos []forgeRepo
if err := s.list(ctx, fmt.Sprintf("/api/v1/teams/%d/repos", t.ID), &haveRepos); err != nil {
return err
}
held := map[string]bool{}
for _, r := range haveRepos {
held[r.FullName] = true
if !repos[r.FullName] {
if err := s.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/teams/%d/repos/%s", t.ID, r.FullName), nil, nil); err != nil {
errs = append(errs, err.Error())
continue
}
s.say("took %s from the team %s", r.FullName, t.Name)
}
}
for _, r := range sortedKeys(repos) {
if held[r] {
continue
}
if err := s.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/repos/%s", t.ID, r), nil, nil); err != nil {
errs = append(errs, fmt.Sprintf("%s could not be given to the team: %v", r, err))
continue
}
s.say("gave %s to the team %s, to read", r, t.Name)
}
var haveMembers []forgeMember
if err := s.list(ctx, fmt.Sprintf("/api/v1/teams/%d/members", t.ID), &haveMembers); err != nil {
return err
}
in := map[string]bool{}
for _, m := range haveMembers {
in[m.Login] = true
if !members[m.Login] {
if err := s.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/teams/%d/members/%s", t.ID, url.PathEscape(m.Login)), nil, nil); err != nil {
errs = append(errs, err.Error())
continue
}
s.say("took %s out of the team %s", m.Login, t.Name)
}
}
for _, m := range sortedKeys(members) {
if in[m] {
continue
}
if err := s.Forge.GetJSON(ctx, "/api/v1/users/"+url.PathEscape(m), nil); err != nil {
if errors.Is(err, forgehttp.ErrNotFound) {
s.say("skipped %s for the team %s: the account does not exist yet", m, t.Name)
continue
}
errs = append(errs, err.Error())
continue
}
if err := s.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/members/%s", t.ID, url.PathEscape(m)), nil, nil); err != nil {
errs = append(errs, fmt.Sprintf("%s could not be put in the team: %v", m, err))
continue
}
s.say("put %s in the team %s", m, t.Name)
}
if len(errs) > 0 {
return errors.New(strings.Join(errs, "; "))
}
return nil
}
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// list reads every page of a list answer into into (a pointer to a slice).
func (s *Syncer) list(ctx context.Context, path string, into any) error {
var all []json.RawMessage
for p := 1; ; p++ {
var got []json.RawMessage
if err := s.Forge.GetJSON(ctx, fmt.Sprintf("%s?limit=%d&page=%d", path, page, p), &got); err != nil {
return err
}
all = append(all, got...)
if len(got) < page {
break
}
}
raw, _ := json.Marshal(all)
return json.Unmarshal(raw, into)
}
func (s *Syncer) teams(ctx context.Context, org string) ([]forgeTeam, error) {
var out []forgeTeam
if err := s.list(ctx, "/api/v1/orgs/"+org+"/teams", &out); err != nil {
if errors.Is(err, forgehttp.ErrNotFound) {
return nil, fmt.Errorf("the organisation %s does not exist on the forge", org)
}
return nil, err
}
return out, nil
}
// orgs are the organisations on the forge, where a read team no longer in the value is found.
func (s *Syncer) orgs(ctx context.Context) ([]string, error) {
var out []struct {
Name string `json:"name"`
}
if err := s.list(ctx, "/api/v1/admin/orgs", &out); err != nil {
return nil, fmt.Errorf("the forge's organisations could not be read: %w", err)
}
names := make([]string, len(out))
for i, o := range out {
names[i] = o.Name
}
return names, nil
}
+333
View File
@@ -0,0 +1,333 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"testing"
"gitea/internal/forgehttp"
)
type fTeam struct {
ID int64
Org string
Name string
Perm string
Units map[string]string
Repos map[string]bool
Members map[string]bool
}
type fakeForge struct {
mu sync.Mutex
orgs []string
users map[string]bool
teams []*fTeam
next int64
}
func newFake(orgs []string, users ...string) *fakeForge {
f := &fakeForge{orgs: orgs, users: map[string]bool{}, next: 1}
for _, u := range users {
f.users[u] = true
}
return f
}
func (f *fakeForge) add(org, name, perm string, repos, members []string) *fTeam {
t := &fTeam{ID: f.next, Org: org, Name: name, Perm: perm, Repos: map[string]bool{}, Members: map[string]bool{}}
f.next++
for _, r := range repos {
t.Repos[r] = true
}
for _, m := range members {
t.Members[m] = true
}
f.teams = append(f.teams, t)
return t
}
func (f *fakeForge) byName(org, name string) *fTeam {
for _, t := range f.teams {
if t.Org == org && t.Name == name {
return t
}
}
return nil
}
var (
orgTeams = regexp.MustCompile(`^/api/v1/orgs/([^/]+)/teams$`)
teamPath = regexp.MustCompile(`^/api/v1/teams/(\d+)$`)
teamRepos = regexp.MustCompile(`^/api/v1/teams/(\d+)/repos(?:/(.+))?$`)
teamMems = regexp.MustCompile(`^/api/v1/teams/(\d+)/members(?:/(.+))?$`)
)
func (f *fakeForge) find(id string) *fTeam {
n, _ := strconv.ParseInt(id, 10, 64)
for _, t := range f.teams {
if t.ID == n {
return t
}
}
return nil
}
func keys(m map[string]bool) []string {
out := []string{}
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
if u, p, ok := r.BasicAuth(); !ok || u != "mesh-admin" || p != "pw" {
http.Error(w, "no", http.StatusUnauthorized)
return
}
reply := func(status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if v != nil {
_ = json.NewEncoder(w).Encode(v)
}
}
paged := r.URL.Query().Get("page") != "" && r.URL.Query().Get("page") != "1"
var in map[string]any
_ = json.NewDecoder(r.Body).Decode(&in)
p := r.URL.Path
switch {
case p == "/api/v1/version":
reply(200, map[string]string{"version": "1"})
case p == "/api/v1/admin/orgs":
out := []map[string]string{}
for _, o := range f.orgs {
out = append(out, map[string]string{"name": o})
}
if paged {
out = nil
}
reply(200, out)
case strings.HasPrefix(p, "/api/v1/users/"):
if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] {
reply(404, nil)
return
}
reply(200, map[string]string{})
case orgTeams.MatchString(p):
org := orgTeams.FindStringSubmatch(p)[1]
known := false
for _, o := range f.orgs {
known = known || o == org
}
if !known {
reply(404, nil)
return
}
if r.Method == http.MethodPost {
units, _ := in["units_map"].(map[string]any)
t := f.add(org, in["name"].(string), in["permission"].(string), nil, nil)
t.Units = map[string]string{}
for k, v := range units {
t.Units[k] = v.(string)
}
reply(201, map[string]any{"id": t.ID, "name": t.Name, "permission": t.Perm})
return
}
out := []map[string]any{}
for _, t := range f.teams {
if t.Org == org && !paged {
out = append(out, map[string]any{"id": t.ID, "name": t.Name, "permission": t.Perm})
}
}
reply(200, out)
case teamPath.MatchString(p):
t := f.find(teamPath.FindStringSubmatch(p)[1])
t.Perm = in["permission"].(string)
reply(200, nil)
case teamRepos.MatchString(p):
m := teamRepos.FindStringSubmatch(p)
t := f.find(m[1])
switch {
case m[2] == "" && r.Method == http.MethodGet:
out := []map[string]string{}
for _, k := range keys(t.Repos) {
out = append(out, map[string]string{"full_name": k})
}
if paged {
out = nil
}
reply(200, out)
case r.Method == http.MethodPut:
t.Repos[m[2]] = true
reply(204, nil)
case r.Method == http.MethodDelete:
delete(t.Repos, m[2])
reply(204, nil)
}
case teamMems.MatchString(p):
m := teamMems.FindStringSubmatch(p)
t := f.find(m[1])
switch {
case m[2] == "" && r.Method == http.MethodGet:
out := []map[string]string{}
for _, k := range keys(t.Members) {
out = append(out, map[string]string{"login": k})
}
if paged {
out = nil
}
reply(200, out)
case r.Method == http.MethodPut:
t.Members[m[2]] = true
reply(204, nil)
case r.Method == http.MethodDelete:
delete(t.Members, m[2])
reply(204, nil)
}
default:
reply(404, nil)
}
}
func syncer(t *testing.T, f *fakeForge) (*Syncer, *[]string) {
t.Helper()
srv := httptest.NewServer(f)
t.Cleanup(srv.Close)
pw := filepath.Join(t.TempDir(), "pw")
if err := os.WriteFile(pw, []byte("pw\n"), 0o600); err != nil {
t.Fatal(err)
}
var said []string
return &Syncer{Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw},
Log: func(format string, a ...any) { said = append(said, sprintf(format, a...)) }}, &said
}
func sprintf(format string, a ...any) string { return fmt.Sprintf(format, a...) }
func equal(a, b []string) bool { return strings.Join(a, ",") == strings.Join(b, ",") }
// The first test of issue 758: a value makes a read-only team with exactly its repositories and members; a changed
// value takes a repository and a member away; a team that is not named agent-read-* is never touched.
func TestValueMakesReadTeamAndChangeRemoves(t *testing.T) {
f := newFake([]string{"novox"}, "agent-g14", "agent-ace")
other := f.add("novox", "Owners", "owner", []string{"novox/hq"}, []string{"agent-ace"})
s, said := syncer(t, f)
ctx := context.Background()
err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq", "novox/mesh-catalog"}, Members: []string{"g14"}}})
if err != nil {
t.Fatal(err)
}
team := f.byName("novox", "agent-read-all")
if team == nil || team.Perm != "read" {
t.Fatalf("team = %+v, want a read team", team)
}
for _, u := range team.Units {
if u != "read" {
t.Errorf("unit %q, want read only", u)
}
}
if !equal(keys(team.Repos), []string{"novox/hq", "novox/mesh-catalog"}) || !equal(keys(team.Members), []string{"agent-g14"}) {
t.Fatalf("repos %v members %v", keys(team.Repos), keys(team.Members))
}
before := len(*said)
if err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq", "novox/mesh-catalog"}, Members: []string{"g14"}}}); err != nil {
t.Fatal(err)
}
if len(*said) != before {
t.Errorf("a second run changed something: %v", (*said)[before:])
}
if err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq"}, Members: nil}}); err != nil {
t.Fatal(err)
}
if !equal(keys(team.Repos), []string{"novox/hq"}) || len(team.Members) != 0 {
t.Fatalf("after change: repos %v members %v", keys(team.Repos), keys(team.Members))
}
// the team leaves the value: emptied, never deleted
if err := s.Sync(ctx, Value{}); err != nil {
t.Fatal(err)
}
if f.byName("novox", "agent-read-all") == nil || len(team.Repos) != 0 || len(team.Members) != 0 {
t.Fatalf("removed team not emptied: %+v", team)
}
if other.Perm != "owner" || !equal(keys(other.Repos), []string{"novox/hq"}) || !equal(keys(other.Members), []string{"agent-ace"}) {
t.Fatalf("a team without the prefix was touched: %+v", other)
}
for _, line := range *said {
if strings.Contains(line, "pw") {
t.Errorf("a line names the password: %q", line)
}
}
}
func TestPermissionIsCorrectedAndMissingAccountSkipped(t *testing.T) {
f := newFake([]string{"novox"}, "agent-g14")
team := f.add("novox", "agent-read-all", "write", nil, nil)
s, said := syncer(t, f)
err := s.Sync(context.Background(), Value{"all": {Repos: []string{"novox/hq"}, Members: []string{"g14", "later"}}})
if err != nil {
t.Fatal(err)
}
if team.Perm != "read" || !equal(keys(team.Members), []string{"agent-g14"}) {
t.Fatalf("team = %+v", team)
}
if !strings.Contains(strings.Join(*said, "\n"), "agent-later") {
t.Errorf("the missing account was not said: %v", *said)
}
}
func TestRefusesWhatIsNotAReadTeam(t *testing.T) {
f := newFake([]string{"novox", "other"})
s, _ := syncer(t, f)
for name, v := range map[string]Value{
"no repo": {"a": {Members: []string{"g14"}}},
"two owners": {"a": {Repos: []string{"novox/hq", "other/x"}}},
"bad repo": {"a": {Repos: []string{"hq"}}},
"bad member": {"a": {Repos: []string{"novox/hq"}, Members: []string{"../x"}}},
"unknown org": {"a": {Repos: []string{"nowhere/hq"}}},
"bad teamname": {"a b": {Repos: []string{"novox/hq"}}},
} {
if err := s.Sync(context.Background(), v); err == nil {
t.Errorf("%s: no error", name)
}
}
if len(f.teams) != 0 {
t.Errorf("teams made: %+v", f.teams)
}
}
func TestReadValue(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "t.json")
_ = os.WriteFile(p, []byte(`{"agent-read-teams":{"a":{"repos":["o/r"],"members":["g14"]}}}`), 0o600)
v, err := ReadValue(p)
if err != nil || len(v["a"].Repos) != 1 || v["a"].Members[0] != "g14" {
t.Fatalf("%v %v", v, err)
}
_ = os.WriteFile(p, []byte(`{}`), 0o600)
if v, err := ReadValue(p); err != nil || len(v) != 0 {
t.Fatalf("%v %v", v, err)
}
_ = os.WriteFile(p, []byte(`{"agent-read-teams":{"a":{"repos":["o/r"],"write":true}}}`), 0o600)
if _, err := ReadValue(p); err == nil {
t.Fatal("an unknown field was accepted")
}
}