Files
nofx/telegram/agent/agent_test.go
tinkle-community 5f47dd13db fix(telegram): eliminate narration, add full-setup workflow and tests
- Rewrite NO NARRATION rule: response is EITHER api_call tag alone OR
  final text reply — no text before api_call under any circumstances
- Ban all narration patterns: 现在我将/好的/正在/I will/Let me etc.
- Add 'create strategy + create trader + start' full setup workflow
- Add 12 automated tests covering:
  - No narration leaking to user (5 narration variants tested)
  - api_call tag never leaks to user
  - Full setup workflow: POST strategy → verify → POST trader → start
  - Start existing trader workflow
  - Max iterations safety, tag stripping, parser edge cases
2026-03-08 16:44:45 +08:00

431 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package agent
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"nofx/mcp"
)
type mockLLM struct {
responses []string
calls int
lastMsgs []mcp.Message
}
func (m *mockLLM) SetAPIKey(_, _, _ string) {}
func (m *mockLLM) SetTimeout(_ time.Duration) {}
func (m *mockLLM) CallWithMessages(_, _ string) (string, error) { return m.next() }
func (m *mockLLM) CallWithRequest(req *mcp.Request) (string, error) {
m.lastMsgs = req.Messages
return m.next()
}
func (m *mockLLM) CallWithRequestStream(req *mcp.Request, onChunk func(string)) (string, error) {
m.lastMsgs = req.Messages
r, err := m.next()
if onChunk != nil {
onChunk(r)
}
return r, err
}
func (m *mockLLM) next() (string, error) {
if m.calls < len(m.responses) {
r := m.responses[m.calls]
m.calls++
return r, nil
}
return "OK", nil
}
func mockGetLLM(llm *mockLLM) func() mcp.AIClient {
return func() mcp.AIClient { return llm }
}
const testPrompt = "You are a test assistant."
// mockAPIServer creates a test HTTP server with configurable route handlers.
func mockAPIServer(handlers map[string]string) (*httptest.Server, int) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
key := r.Method + " " + r.URL.Path
if body, ok := handlers[key]; ok {
w.Write([]byte(body)) //nolint:errcheck
return
}
// Also try path-only match (for GET)
if body, ok := handlers[r.URL.Path]; ok {
w.Write([]byte(body)) //nolint:errcheck
return
}
w.WriteHeader(http.StatusNotFound)
w.Write([]byte(`{"error":"not found"}`)) //nolint:errcheck
}))
var port int
fmt.Sscanf(srv.Listener.Addr().String(), "127.0.0.1:%d", &port)
return srv, port
}
// ── Basic agent behaviour ──────────────────────────────────────────────────
// TestAgentDirectReply: LLM replies without api_call — one call, direct reply.
func TestAgentDirectReply(t *testing.T) {
llm := &mockLLM{responses: []string{"Hello! How can I help you?"}}
a := New(8080, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("hello", nil)
if reply != "Hello! How can I help you?" {
t.Fatalf("unexpected reply: %q", reply)
}
if llm.calls != 1 {
t.Fatalf("expected 1 LLM call, got %d", llm.calls)
}
}
// TestAgentAPICall: LLM calls API, gets result, gives final reply — two LLM calls.
func TestAgentAPICall(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/my-traders": `[{"trader_id":"t1","trader_name":"BTC Trader","is_running":false}]`,
})
defer srv.Close()
llm := &mockLLM{responses: []string{
`<api_call>{"method":"GET","path":"/api/my-traders","body":{}}</api_call>`,
"You have one trader: BTC Trader.",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("list my traders", nil)
if reply != "You have one trader: BTC Trader." {
t.Fatalf("unexpected reply: %q", reply)
}
if llm.calls != 2 {
t.Fatalf("expected 2 LLM calls, got %d", llm.calls)
}
}
// TestAgentMultiStep: LLM chains two API calls before final reply — three LLM calls.
func TestAgentMultiStep(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/account": `{"total_equity":1000}`,
"/api/positions": `[]`,
})
defer srv.Close()
llm := &mockLLM{responses: []string{
`<api_call>{"method":"GET","path":"/api/account","body":{}}</api_call>`,
`<api_call>{"method":"GET","path":"/api/positions","body":{}}</api_call>`,
"Account looks healthy and no open positions.",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("show me account status", nil)
if llm.calls != 3 {
t.Fatalf("expected 3 LLM calls (2 api + 1 final), got %d", llm.calls)
}
if reply != "Account looks healthy and no open positions." {
t.Fatalf("unexpected final reply: %q", reply)
}
}
// TestAgentAPIResultInContext: API result must appear in next LLM message.
func TestAgentAPIResultInContext(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/account": `{"balance":1234.56}`,
})
defer srv.Close()
llm := &mockLLM{responses: []string{
`<api_call>{"method":"GET","path":"/api/account","body":{}}</api_call>`,
"Balance is 1234.56 USDT.",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
a.Run("show balance", nil)
found := false
for _, msg := range llm.lastMsgs {
if strings.Contains(msg.Content, "API result") || strings.Contains(msg.Content, "balance") {
found = true
break
}
}
if !found {
t.Fatalf("API result not found in subsequent LLM context")
}
}
// ── NO NARRATION tests ─────────────────────────────────────────────────────
// TestNoNarrationBeforeAPICall: any text before <api_call> must NOT reach the user.
// The agent strips text-before-tag and only forwards it as assistant context.
func TestNoNarrationBeforeAPICall(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/strategies": `[{"id":"s1","name":"BTC Trend"}]`,
})
defer srv.Close()
narrations := []string{
"现在我将为您创建策略。\n",
"好的,我来帮你查询。",
"Let me check this for you. ",
"正在处理...",
"I will call the API now. ",
}
for _, narration := range narrations {
llm := &mockLLM{responses: []string{
// LLM outputs narration before the api_call tag (bad behaviour we must handle)
narration + `<api_call>{"method":"GET","path":"/api/strategies","body":{}}</api_call>`,
"你有1个策略BTC Trend。",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("查询我的策略", nil)
// Final reply must not contain narration fragments
if strings.Contains(reply, "现在我将") || strings.Contains(reply, "Let me") ||
strings.Contains(reply, "正在处理") || strings.Contains(reply, "好的,我来") ||
strings.Contains(reply, "I will call") {
t.Fatalf("narration leaked into reply for input %q: got %q", narration, reply)
}
// api_call tag must not appear in reply
if strings.Contains(reply, "<api_call>") {
t.Fatalf("api_call tag leaked into reply: %q", reply)
}
}
}
// TestAPICallTagNotLeakedToUser: <api_call> tag must never appear in returned reply.
func TestAPICallTagNotLeakedToUser(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/account": `{"total_equity":500}`,
})
defer srv.Close()
llm := &mockLLM{responses: []string{
`<api_call>{"method":"GET","path":"/api/account","body":{}}</api_call>`,
`账户余额 500 USDT。<api_call>{"method":"GET","path":"/api/account","body":{}}</api_call>`,
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("show balance", nil)
if strings.Contains(reply, "<api_call>") {
t.Fatalf("api_call tag leaked to user: %q", reply)
}
}
// ── Workflow tests ─────────────────────────────────────────────────────────
// TestCreateStrategyWorkflow: simulates creating a BTC trend strategy.
// Verifies: POST strategy → GET verify → final reply shows strategy info.
func TestCreateStrategyWorkflow(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"POST /api/strategies": `{"id":"s1","name":"BTC趋势"}`,
"GET /api/strategies/s1": `{"id":"s1","name":"BTC趋势","config":{"coin_source":{"source_type":"static","static_coins":["BTC/USDT"]},"leverage":5}}`,
})
defer srv.Close()
llm := &mockLLM{responses: []string{
// Step 1: create strategy
`<api_call>{"method":"POST","path":"/api/strategies","body":{"name":"BTC趋势","config":{}}}</api_call>`,
// Step 2: verify strategy
`<api_call>{"method":"GET","path":"/api/strategies/s1","body":{}}</api_call>`,
// Step 3: final reply
"策略已创建BTC趋势币种 BTC/USDT杠杆 5x。",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("帮我配置个btc趋势交易的策略", nil)
if llm.calls != 3 {
t.Fatalf("expected 3 LLM calls, got %d", llm.calls)
}
if reply == "" || strings.Contains(reply, "<api_call>") {
t.Fatalf("bad final reply: %q", reply)
}
}
// TestFullSetupWorkflow: create strategy → create trader → start trader.
// This is the "帮我配置策略并跑起来" workflow.
func TestFullSetupWorkflow(t *testing.T) {
calls := map[string]int{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
key := r.Method + " " + r.URL.Path
calls[key]++
switch key {
case "POST /api/strategies":
w.Write([]byte(`{"id":"s1","name":"BTC趋势"}`)) //nolint:errcheck
case "GET /api/strategies/s1":
w.Write([]byte(`{"id":"s1","name":"BTC趋势","config":{}}`)) //nolint:errcheck
case "POST /api/traders":
w.Write([]byte(`{"id":"tr1","name":"BTC趋势交易员"}`)) //nolint:errcheck
case "POST /api/traders/tr1/start":
w.Write([]byte(`{"ok":true}`)) //nolint:errcheck
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
var port int
fmt.Sscanf(srv.Listener.Addr().String(), "127.0.0.1:%d", &port)
llm := &mockLLM{responses: []string{
// 1. create strategy
`<api_call>{"method":"POST","path":"/api/strategies","body":{"name":"BTC趋势"}}</api_call>`,
// 2. verify strategy
`<api_call>{"method":"GET","path":"/api/strategies/s1","body":{}}</api_call>`,
// 3. create trader
`<api_call>{"method":"POST","path":"/api/traders","body":{"name":"BTC趋势交易员","strategy_id":"s1"}}</api_call>`,
// 4. start trader
`<api_call>{"method":"POST","path":"/api/traders/tr1/start","body":{}}</api_call>`,
// 5. final reply
"策略和交易员已创建并启动BTC趋势交易员正在运行。",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("帮我配置个btc趋势交易的策略交易 跑起来", nil)
if llm.calls != 5 {
t.Fatalf("expected 5 LLM calls, got %d", llm.calls)
}
// Verify each API was called
if calls["POST /api/strategies"] != 1 {
t.Errorf("expected 1 POST /api/strategies, got %d", calls["POST /api/strategies"])
}
if calls["POST /api/traders"] != 1 {
t.Errorf("expected 1 POST /api/traders, got %d", calls["POST /api/traders"])
}
if calls["POST /api/traders/tr1/start"] != 1 {
t.Errorf("expected 1 POST /api/traders/tr1/start, got %d", calls["POST /api/traders/tr1/start"])
}
if strings.Contains(reply, "<api_call>") {
t.Fatalf("api_call tag in final reply: %q", reply)
}
}
// TestStartExistingTrader: when trader already exists, just start it.
func TestStartExistingTrader(t *testing.T) {
calls := map[string]int{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
key := r.Method + " " + r.URL.Path
calls[key]++
switch key {
case "GET /api/my-traders":
w.Write([]byte(`[{"trader_id":"tr1","trader_name":"BTC Trader","is_running":false}]`)) //nolint:errcheck
case "POST /api/traders/tr1/start":
w.Write([]byte(`{"ok":true}`)) //nolint:errcheck
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
var port int
fmt.Sscanf(srv.Listener.Addr().String(), "127.0.0.1:%d", &port)
llm := &mockLLM{responses: []string{
`<api_call>{"method":"GET","path":"/api/my-traders","body":{}}</api_call>`,
`<api_call>{"method":"POST","path":"/api/traders/tr1/start","body":{}}</api_call>`,
"交易员 BTC Trader 已启动。",
}}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("启动交易员", nil)
if calls["POST /api/traders/tr1/start"] != 1 {
t.Errorf("expected trader to be started, got %d start calls", calls["POST /api/traders/tr1/start"])
}
if strings.Contains(reply, "<api_call>") {
t.Fatalf("api_call tag in reply: %q", reply)
}
}
// ── Parser tests ───────────────────────────────────────────────────────────
// TestParseAPICall: unit tests for the XML tag parser.
func TestParseAPICall(t *testing.T) {
t.Run("valid call no text before", func(t *testing.T) {
resp := `<api_call>{"method":"POST","path":"/api/traders/t1/stop","body":{}}</api_call>`
req, text := parseAPICall(resp)
if req == nil {
t.Fatal("expected api_call, got nil")
}
if req.Method != "POST" || req.Path != "/api/traders/t1/stop" {
t.Fatalf("unexpected req: %+v", req)
}
if text != "" {
t.Fatalf("expected empty text before tag, got: %q", text)
}
})
t.Run("text before tag is captured", func(t *testing.T) {
resp := `Stopping trader.<api_call>{"method":"POST","path":"/api/traders/t1/stop","body":{}}</api_call>`
req, text := parseAPICall(resp)
if req == nil {
t.Fatal("expected api_call, got nil")
}
if text != "Stopping trader." {
t.Fatalf("unexpected text before tag: %q", text)
}
})
t.Run("no call tag", func(t *testing.T) {
req, text := parseAPICall("Just a reply.")
if req != nil {
t.Fatal("expected nil api_call")
}
if text != "Just a reply." {
t.Fatalf("expected original text, got %q", text)
}
})
t.Run("malformed JSON", func(t *testing.T) {
req, _ := parseAPICall(`<api_call>NOT JSON</api_call>`)
if req != nil {
t.Fatal("expected nil for malformed JSON")
}
})
}
// TestStripAPICallTag: defensive cleanup of stray tags in final reply.
func TestStripAPICallTag(t *testing.T) {
cases := []struct {
input string
want string
}{
{`正常回复`, `正常回复`},
{`回复<api_call>{"method":"GET","path":"/x"}</api_call>`, `回复`},
{`<api_call>{"method":"GET","path":"/x"}</api_call>`, ``},
}
for _, c := range cases {
got := stripAPICallTag(c.input)
if strings.TrimSpace(got) != c.want {
t.Errorf("stripAPICallTag(%q) = %q, want %q", c.input, got, c.want)
}
}
}
// TestMaxIterations: agent stops after maxIterations and returns a summary.
func TestMaxIterations(t *testing.T) {
srv, port := mockAPIServer(map[string]string{
"/api/account": `{"ok":true}`,
})
defer srv.Close()
// Always returns another api_call — should hit max iterations
responses := make([]string, maxIterations+2)
for i := range responses {
responses[i] = `<api_call>{"method":"GET","path":"/api/account","body":{}}</api_call>`
}
responses[maxIterations] = "Final summary after max iterations."
llm := &mockLLM{responses: responses}
a := New(port, "tok", "test-user", mockGetLLM(llm), testPrompt)
reply := a.Run("loop forever", nil)
if strings.Contains(reply, "<api_call>") {
t.Fatalf("api_call tag in reply after max iterations: %q", reply)
}
_ = reply // just confirm it terminates
}