package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "regexp" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/provider" "gitea/internal/forgehttp" ) const ( pubA = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" pubB = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" ) // fakeBus is the bus as the provider reads and writes it. type fakeBus struct { mu sync.Mutex buckets map[string]map[string][]byte } func newBus() *fakeBus { return &fakeBus{buckets: map[string]map[string][]byte{}} } func (b *fakeBus) set(bucket, key string, v []byte) { b.mu.Lock() defer b.mu.Unlock() if b.buckets[bucket] == nil { b.buckets[bucket] = map[string][]byte{} } b.buckets[bucket][key] = v } func (b *fakeBus) Watch(_ context.Context, bucket string, h func(string, []byte)) error { b.mu.Lock() cur := map[string][]byte{} for k, v := range b.buckets[bucket] { cur[k] = v } b.mu.Unlock() for k, v := range cur { h(k, v) } return nil } func (b *fakeBus) Put(_ context.Context, bucket, key string, v []byte) error { b.set(bucket, key, v) return nil } func (b *fakeBus) Delete(_ context.Context, bucket, key string) error { b.mu.Lock() defer b.mu.Unlock() delete(b.buckets[bucket], key) return nil } func (b *fakeBus) entry(t *testing.T, key string) provider.Entry { t.Helper() b.mu.Lock() raw := b.buckets[provider.BucketOf("gitea")][key] b.mu.Unlock() if raw == nil { t.Fatalf("no entry %s", key) } var e provider.Entry if err := json.Unmarshal(raw, &e); err != nil { t.Fatal(err) } return e } func (b *fakeBus) has(key string) bool { b.mu.Lock() defer b.mu.Unlock() _, ok := b.buckets[provider.BucketOf("gitea")][key] return ok } // bindClient makes ssh-client on node bind gitea's forge-account in the assignments mirror. func (b *fakeBus) bindClient(node string) { b.set(provider.AssignmentsBucket, node+".ssh-client", []byte(fmt.Sprintf( `{"node":%q,"module":"ssh-client","provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s_ssh_client"}]}}`, node, node))) } // fakeForge is gitea's API as far as the provider uses it. type fakeForge struct { mu sync.Mutex users map[string]bool teams map[string]bool members []string held map[string][]map[string]any next int64 } func newForge() *fakeForge { return &fakeForge{users: map[string]bool{}, held: map[string][]map[string]any{}} } var keyRe = regexp.MustCompile(`^/api/v1/(?:admin/)?users/([^/]+)/keys(?:/(\d+))?$`) func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() body, _ := io.ReadAll(r.Body) var in map[string]any _ = json.Unmarshal(body, &in) reply := func(code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) if v != nil { _ = json.NewEncoder(w).Encode(v) } } p := r.URL.Path if m := keyRe.FindStringSubmatch(p); m != nil { switch r.Method { case http.MethodGet: reply(200, f.held[m[1]]) case http.MethodPost: f.next++ f.held[m[1]] = append(f.held[m[1]], map[string]any{"id": f.next, "title": in["title"], "key": in["key"]}) reply(201, map[string]any{"id": f.next}) case http.MethodDelete: var kept []map[string]any for _, k := range f.held[m[1]] { if fmt.Sprint(k["id"]) != m[2] { kept = append(kept, k) } } f.held[m[1]] = kept reply(204, nil) } return } switch { case r.Method == http.MethodGet && strings.HasPrefix(p, "/api/v1/users/"): if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] { reply(404, map[string]any{"message": "no"}) return } reply(200, map[string]any{"active": true}) case r.Method == http.MethodPost && p == "/api/v1/admin/users": f.users[in["username"].(string)] = true reply(201, map[string]any{}) case p == "/api/v1/orgs/novox/teams" && r.Method == http.MethodGet: reply(200, []map[string]any{{"id": 5, "name": "agents"}}) case p == "/api/v1/teams/5" && r.Method == http.MethodPatch: reply(200, map[string]any{}) case strings.HasPrefix(p, "/api/v1/teams/5/members/") && r.Method == http.MethodPut: f.members = append(f.members, strings.TrimPrefix(p, "/api/v1/teams/5/members/")) reply(204, nil) default: http.Error(w, "unexpected "+r.Method+" "+p, 400) } } func (f *fakeForge) titles(user string) []string { f.mu.Lock() defer f.mu.Unlock() var out []string for _, k := range f.held[user] { out = append(out, k["title"].(string)) } return out } type fakeRunner struct { out string err error args []string } func (r *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { r.args = append([]string{name}, args...) return []byte(r.out), r.err } const scanned = "# 127.0.0.1:222 SSH-2.0-Go\n[127.0.0.1]:222 ssh-ed25519 AAAAC3HOSTKEY\n" type rig struct { t *testing.T bus *fakeBus forge *fakeForge svc *Service runner *fakeRunner } func newRig(t *testing.T) *rig { t.Helper() forge := newForge() srv := httptest.NewServer(forge) t.Cleanup(srv.Close) dir := t.TempDir() pw := filepath.Join(dir, "pw") host := filepath.Join(dir, "host") origins := filepath.Join(dir, "origins") for p, c := range map[string]string{pw: "x\n", host: "novox.internal\n", origins: "https://git.novox.be\n"} { if err := os.WriteFile(p, []byte(c), 0o600); err != nil { t.Fatal(err) } } runner := &fakeRunner{out: scanned} svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222, Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}}) svc.loaded = true // the machines' keys are read; a test that means otherwise says so return &rig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner} } func (r *rig) pass() []provider.Outcome { r.t.Helper() p := r.svc.Provider(r.bus) out, err := p.Pass(context.Background()) if err != nil { r.t.Fatal(err) } return out } func (r *rig) key(node, account, public string) { b, _ := json.Marshal(map[string]string{"node": node, "account": account, "kind": "machine", "public": public, "fingerprint": "SHA256:x", "at": "2026-10-16T10:00:00Z"}) r.svc.KeyChanged(node+"."+account, b) } func TestAConsumerWithAKeyGetsItsEntryAndItsKeyIsRegistered(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("g14", "nox", pubA+" nox@g14 (mesh)") r.pass() e := r.bus.entry(t, "g14.ssh-client.forge-account") want := map[string]any{"user": "agent-g14", "ssh_host": "novox.internal", "ssh_port": float64(222), "known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY", "ssh_alias": "mesh-forge", "host_key": "ssh-ed25519 AAAAC3HOSTKEY", "ssh_base": "ssh://git@novox.internal:222/", "https_base": "https://git.novox.be/"} for k, v := range want { if e.Values[k] != v { t.Errorf("%s = %v, want %v", k, e.Values[k], v) } } if e.VaultName != "" || e.User != "" { t.Errorf("a public key is no secret; the entry names a credential: %+v", e) } if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "mesh:g14.nox" { t.Errorf("registered keys = %v", got) } if !r.forge.users["agent-g14"] || len(r.forge.members) != 1 || r.forge.members[0] != "agent-g14" { t.Errorf("account %v, team members %v", r.forge.users, r.forge.members) } if strings.Join(r.runner.args, " ") != "ssh-keyscan -t ed25519 -p 222 127.0.0.1" { t.Errorf("scan = %v", r.runner.args) } } func TestAConsumerWithoutAKeyGetsNoEntryAndNoAccount(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("ace", "nox", pubB) // another machine's key is not g14's out := r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("an entry was written for a machine that reported no key") } if len(r.forge.users) != 0 { t.Fatalf("an account was made: %v", r.forge.users) } if len(out) != 1 || !strings.Contains(out[0].Why, "no key") { t.Fatalf("the wait should be said: %+v", out) } } func TestAKeyOfAnotherKindIsNotTheMachinesKey(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") b, _ := json.Marshal(map[string]string{"node": "g14", "account": "x", "kind": "session", "public": pubA}) r.svc.KeyChanged("g14.x", b) r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("answered with a key that is not a machine's") } } func TestAChangedKeyReplacesTheOldMeshKey(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("g14", "nox", pubA+" nox@g14 (mesh)") r.pass() r.forge.mu.Lock() r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() r.key("g14", "nox", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC nox@g14 (mesh)") r.pass() got := strings.Join(r.forge.titles("agent-g14"), ",") if got != "my own,mesh:g14.nox" { t.Fatalf("keys = %s", got) } for _, k := range r.forge.held["agent-g14"] { if k["title"] == "mesh:g14.nox" && !strings.Contains(k["key"].(string), "CCCC") { t.Fatalf("the old key stayed: %v", k) } } } func TestAConsumerThatNoLongerBindsLosesItsMeshKeysAndKeepsItsAccount(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("g14", "nox", pubA+" nox@g14 (mesh)") r.pass() r.forge.mu.Lock() r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() r.bus.mu.Lock() delete(r.bus.buckets[provider.AssignmentsBucket], "g14.ssh-client") r.bus.mu.Unlock() r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("the entry stayed") } if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "my own" { t.Fatalf("keys = %v", got) } if !r.forge.users["agent-g14"] { t.Fatal("the account was removed") } } func TestAKeyChangeNamesTheConsumersToAnswerAgain(t *testing.T) { r := newRig(t) var got []string r.svc.OnKeys = func(node string) { got = append(got, node) } r.key("g14", "nox", pubA) if len(got) != 1 || got[0] != "g14" { t.Fatalf("answered for %v", got) } } func TestAKeyThatIsNotReadableIsNotKept(t *testing.T) { r := newRig(t) r.svc.KeyChanged("g14.nox", []byte(`{"node":"g14","kind":"machine"`)) r.svc.KeyChanged("g14.nox", []byte(`{"node":"g14","account":"nox","kind":"machine","public":"PRIVATE KEY"}`)) r.bus.bindClient("g14") r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("a value that is no public key was registered") } } func TestTheForgesHostKeyUnreadableMeansNoEntry(t *testing.T) { r := newRig(t) r.runner.err = errors.New("connection refused") r.bus.bindClient("g14") r.key("g14", "nox", pubA) r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("an entry without the forge's host key") } } func TestKnownHostLineIsRewrittenToTheMeshName(t *testing.T) { line, err := knownHostLine(scanned, "novox.internal", 222) if err != nil || line != "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY" { t.Fatalf("%q %v", line, err) } if _, err := knownHostLine("# nothing\n", "novox.internal", 222); err == nil { t.Fatal("no key line was accepted") } } func TestKeysWatchedRetriesUntilItAnswersAndPassesOnChanges(t *testing.T) { r := newRig(t) src := &fakeKeys{failFirst: 2} r.svc.Pause = func(time.Duration) {} r.svc.loaded = false passes := 0 r.svc.OnKeys = func(string) { passes++ } if err := r.svc.WatchKeys(context.Background(), src); err != nil { t.Fatal(err) } if !r.svc.loaded { t.Fatal("the keys are not loaded once the watch has answered") } if src.calls != 3 { t.Fatalf("asked %d times", src.calls) } src.send("g14.nox", "put", []byte(`{"node":"g14","account":"nox","kind":"machine","public":"`+pubA+`"}`), false) if passes != 1 { t.Fatalf("passes = %d", passes) } // What was there when the watch began is taken in, and one pass answers it, not one per key. src.send("ace.nox", "put", []byte(`{"node":"ace","account":"nox","kind":"machine","public":"`+pubB+`"}`), true) if passes != 1 { t.Fatalf("a current value passed on its own: %d", passes) } src.send("g14.nox", "delete", nil, false) if passes != 2 || len(r.svc.keysOf("g14")) != 0 { t.Fatalf("passes = %d, keys = %v", passes, r.svc.keysOf("g14")) } } type fakeKeys struct { failFirst int calls int handle func(key, op string, value []byte, current bool) } func (f *fakeKeys) Watch(handle func(key, op string, value []byte, current bool)) error { f.calls++ if f.calls <= f.failFirst { return errors.New("not yet") } f.handle = handle return nil } func (f *fakeKeys) send(key, op string, v []byte, current bool) { f.handle(key, op, v, current) } func TestADeletedKeyEntryRemovesTheMeshKeyAndTheBoundEntryButNotTheOperatorsKey(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("g14", "nox", pubA+" nox@g14 (mesh)") r.pass() r.forge.mu.Lock() r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() r.svc.KeyGone("g14.nox") r.pass() if got := r.forge.titles("agent-g14"); len(got) != 1 || got[0] != "my own" { t.Fatalf("keys = %v", got) } if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("the entry still serves a machine whose key is gone") } } func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) { r := newRig(t) r.bus.bindClient("g14") r.key("g14", "nox", pubA+" nox@g14 (mesh)") r.pass() r.forge.mu.Lock() r.forge.held["agent-g14"] = append(r.forge.held["agent-g14"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() // A restart: the keys are not read yet, so a node with none is not a node that reported none. r.svc.mu.Lock() r.svc.keys = map[string]machineKey{} r.svc.loaded = false r.svc.mu.Unlock() out := r.pass() if got := strings.Join(r.forge.titles("agent-g14"), ","); got != "mesh:g14.nox,my own" { t.Fatalf("keys = %s", got) } if !r.bus.has("g14.ssh-client.forge-account") { t.Fatal("the entry was deleted before the keys were read") } if len(out) != 1 || !strings.Contains(out[0].Why, "not read yet") { t.Fatalf("the wait should be said: %+v", out) } // Loaded, and the node reports none: now its keys go. r.svc.mu.Lock() r.svc.loaded = true r.svc.mu.Unlock() r.pass() if got := strings.Join(r.forge.titles("agent-g14"), ","); got != "my own" || r.bus.has("g14.ssh-client.forge-account") { t.Fatalf("keys = %s", got) } } // Issue 955: a watch the runtime refuses (a grant that comes later) must not end the provider for good. func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) { r := newRig(t) var pauses []time.Duration var said []string r.svc.Pause = func(d time.Duration) { pauses = append(pauses, d) } r.svc.cfg.Log = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } ctx, stop := context.WithCancel(context.Background()) defer stop() calls := 0 serving := make(chan struct{}) done := make(chan struct{}) go func() { defer close(done) r.svc.KeepRunning(ctx, func(ctx context.Context) error { calls++ if calls <= 2 { return errors.New(`watching mesh-controller_assignments: gitea reads no state called "mesh-controller.assignments"`) } close(serving) <-ctx.Done() return ctx.Err() }) }() select { case <-serving: case <-time.After(5 * time.Second): t.Fatal("the provider was not started again after its watch was refused") } stop() <-done if calls != 3 { t.Fatalf("run %d times", calls) } if len(pauses) != 2 || pauses[0] != 5*time.Second || pauses[1] != 10*time.Second { t.Fatalf("the backoff is %v", pauses) } if len(said) == 0 || !strings.Contains(said[0], "mesh-controller.assignments") { t.Fatalf("a refusal is said loudly: %v", said) } loud := 0 for _, s := range said { if strings.Contains(s, "stopped") { loud++ } } if loud != 1 { t.Fatalf("the same refusal is said %d times in one spell: %v", loud, said) } } func TestKeepRunningBacksOffToTwoMinutes(t *testing.T) { r := newRig(t) var pauses []time.Duration r.svc.Pause = func(d time.Duration) { pauses = append(pauses, d) } ctx, stop := context.WithCancel(context.Background()) calls := 0 r.svc.KeepRunning(ctx, func(context.Context) error { if calls++; calls == 9 { stop() } return errors.New("refused") }) last := pauses[len(pauses)-1] if pauses[0] != 5*time.Second || last != 2*time.Minute { t.Fatalf("the backoff is %v", pauses) } }