diff --git a/README.md b/README.md index 5741acfd..369f695b 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,6 @@ Also compatible with **[ClawRouter](https://github.com/BlockRunAI/ClawRouter)** | **Strategy Studio** | Visual builder — coin sources, indicators, risk controls | | **AI Competition** | AIs compete in real-time, leaderboard ranks performance | | **Telegram Agent** | Chat with your trading assistant — streaming, tool calling, memory | -| **Backtest Lab** | Historical simulation with equity curves and performance metrics | | **Dashboard** | Live positions, P/L, AI decision logs with Chain of Thought | ### Markets @@ -158,11 +157,11 @@ Crypto · US Stocks · Forex · Metals
-Competition & Backtest +Competition -| Competition Mode | Backtest Lab | -|:---:|:---:| -| | | +| Competition Mode | +|:---:| +| |
--- @@ -253,8 +252,8 @@ curl -fsSL https://raw.githubusercontent.com/NoFxAiOS/nofx/main/install.sh | bas ├─────────────────────────────────────────────────┤ │ API Server (Go) │ ├──────────┬──────────┬──────────┬────────────────┤ - │ Strategy │ Backtest │ Telegram │ - │ Engine │ Lab │ Agent │ + │ Strategy │ Telegram │ + │ Engine │ Agent │ ├──────────┴──────────┴──────────┴────────────────┤ │ MCP AI Client Layer │ │ ┌───────────┐ ┌───────────┐ ┌───────────┐ │ @@ -277,7 +276,6 @@ curl -fsSL https://raw.githubusercontent.com/NoFxAiOS/nofx/main/install.sh | bas |:--|:--| | [Architecture](docs/architecture/README.md) | System design and module index | | [Strategy Module](docs/architecture/STRATEGY_MODULE.md) | Coin selection, AI prompts, execution | -| [Backtest Module](docs/architecture/BACKTEST_MODULE.md) | Historical simulation, metrics | | [FAQ](docs/faq/README.md) | Common questions | | [Getting Started](docs/getting-started/README.md) | Deployment guide | diff --git a/api/backtest.go b/api/backtest.go deleted file mode 100644 index 82b5eecb..00000000 --- a/api/backtest.go +++ /dev/null @@ -1,863 +0,0 @@ -package api - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "net/http" - "os" - "strconv" - "strings" - "time" - - "nofx/backtest" - "nofx/logger" - "nofx/market" - "nofx/provider/nofxos" - "nofx/store" - - "github.com/gin-gonic/gin" -) - -func (s *Server) registerBacktestRoutes(router *gin.RouterGroup) { - router.POST("/start", s.handleBacktestStart) - router.POST("/pause", s.handleBacktestPause) - router.POST("/resume", s.handleBacktestResume) - router.POST("/stop", s.handleBacktestStop) - router.POST("/label", s.handleBacktestLabel) - router.POST("/delete", s.handleBacktestDelete) - router.GET("/status", s.handleBacktestStatus) - router.GET("/runs", s.handleBacktestRuns) - router.GET("/equity", s.handleBacktestEquity) - router.GET("/trades", s.handleBacktestTrades) - router.GET("/metrics", s.handleBacktestMetrics) - router.GET("/trace", s.handleBacktestTrace) - router.GET("/decisions", s.handleBacktestDecisions) - router.GET("/export", s.handleBacktestExport) - router.GET("/klines", s.handleBacktestKlines) -} - -type backtestStartRequest struct { - Config backtest.BacktestConfig `json:"config"` -} - -type runIDRequest struct { - RunID string `json:"run_id"` -} - -type labelRequest struct { - RunID string `json:"run_id"` - Label string `json:"label"` -} - -func (s *Server) handleBacktestStart(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - - var req backtestStartRequest - if err := c.ShouldBindJSON(&req); err != nil { - SafeBadRequest(c, "Invalid request parameters") - return - } - - cfg := req.Config - if cfg.RunID == "" { - cfg.RunID = "bt_" + time.Now().UTC().Format("20060102_150405") - } - cfg.CustomPrompt = strings.TrimSpace(cfg.CustomPrompt) - cfg.UserID = normalizeUserID(c.GetString("user_id")) - - logger.Infof("📊 Backtest request - symbols from request: %v (count=%d), strategyID: %s", - cfg.Symbols, len(cfg.Symbols), cfg.StrategyID) - - // Load strategy config if strategy_id is provided - if cfg.StrategyID != "" { - strategy, err := s.store.Strategy().Get(cfg.UserID, cfg.StrategyID) - if err != nil { - SafeBadRequest(c, "Failed to load strategy") - return - } - if strategy == nil { - SafeBadRequest(c, "Strategy not found") - return - } - var strategyConfig store.StrategyConfig - if err := json.Unmarshal([]byte(strategy.Config), &strategyConfig); err != nil { - SafeBadRequest(c, "Failed to parse strategy config") - return - } - cfg.SetLoadedStrategy(&strategyConfig) - logger.Infof("📊 Backtest using saved strategy: %s (%s)", strategy.Name, strategy.ID) - logger.Infof("📊 Strategy coin source: type=%s, use_ai500=%v, use_oi_top=%v, static_coins=%v", - strategyConfig.CoinSource.SourceType, - strategyConfig.CoinSource.UseAI500, - strategyConfig.CoinSource.UseOITop, - strategyConfig.CoinSource.StaticCoins) - - // If no symbols provided, fetch from strategy's coin source - if len(cfg.Symbols) == 0 { - symbols, err := s.resolveStrategyCoins(&strategyConfig) - if err != nil { - SafeBadRequest(c, "Failed to resolve coins from strategy") - return - } - cfg.Symbols = symbols - logger.Infof("📊 Resolved %d coins from strategy: %v", len(symbols), symbols) - } - } - - if err := s.hydrateBacktestAIConfig(&cfg); err != nil { - SafeBadRequest(c, "Failed to configure AI model") - return - } - - logger.Infof("📊 Starting backtest with final config: runID=%s, symbols=%v (count=%d), strategyID=%s", - cfg.RunID, cfg.Symbols, len(cfg.Symbols), cfg.StrategyID) - - runner, err := s.backtestManager.Start(context.Background(), cfg) - if err != nil { - SafeError(c, http.StatusBadRequest, "Failed to start backtest", err) - return - } - - meta := runner.CurrentMetadata() - c.JSON(http.StatusOK, meta) -} - -func (s *Server) handleBacktestPause(c *gin.Context) { - s.handleBacktestControl(c, s.backtestManager.Pause) -} - -func (s *Server) handleBacktestResume(c *gin.Context) { - s.handleBacktestControl(c, s.backtestManager.Resume) -} - -func (s *Server) handleBacktestStop(c *gin.Context) { - s.handleBacktestControl(c, s.backtestManager.Stop) -} - -func (s *Server) handleBacktestControl(c *gin.Context, fn func(string) error) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - userID := normalizeUserID(c.GetString("user_id")) - - var req runIDRequest - if err := c.ShouldBindJSON(&req); err != nil { - SafeBadRequest(c, "Invalid request parameters") - return - } - if req.RunID == "" { - SafeBadRequest(c, "run_id is required") - return - } - - if _, err := s.ensureBacktestRunOwnership(req.RunID, userID); writeBacktestAccessError(c, err) { - return - } - - if err := fn(req.RunID); err != nil { - SafeError(c, http.StatusBadRequest, "Failed to execute backtest operation", err) - return - } - - meta, err := s.backtestManager.LoadMetadata(req.RunID) - if err != nil { - c.JSON(http.StatusOK, gin.H{"message": "ok"}) - return - } - c.JSON(http.StatusOK, meta) -} - -func (s *Server) handleBacktestLabel(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - var req labelRequest - if err := c.ShouldBindJSON(&req); err != nil { - SafeBadRequest(c, "Invalid request parameters") - return - } - if strings.TrimSpace(req.RunID) == "" { - SafeBadRequest(c, "run_id is required") - return - } - userID := normalizeUserID(c.GetString("user_id")) - if _, err := s.ensureBacktestRunOwnership(req.RunID, userID); writeBacktestAccessError(c, err) { - return - } - meta, err := s.backtestManager.UpdateLabel(req.RunID, req.Label) - if err != nil { - SafeInternalError(c, "Update backtest label", err) - return - } - c.JSON(http.StatusOK, meta) -} - -func (s *Server) handleBacktestDelete(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - var req runIDRequest - if err := c.ShouldBindJSON(&req); err != nil { - SafeBadRequest(c, "Invalid request parameters") - return - } - if strings.TrimSpace(req.RunID) == "" { - SafeBadRequest(c, "run_id is required") - return - } - userID := normalizeUserID(c.GetString("user_id")) - if _, err := s.ensureBacktestRunOwnership(req.RunID, userID); writeBacktestAccessError(c, err) { - return - } - if err := s.backtestManager.Delete(req.RunID); err != nil { - SafeInternalError(c, "Delete backtest run", err) - return - } - c.JSON(http.StatusOK, gin.H{"message": "deleted"}) -} - -func (s *Server) handleBacktestStatus(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - - userID := normalizeUserID(c.GetString("user_id")) - - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - - meta, err := s.ensureBacktestRunOwnership(runID, userID) - if writeBacktestAccessError(c, err) { - return - } - - status := s.backtestManager.Status(runID) - if status != nil { - c.JSON(http.StatusOK, status) - return - } - - payload := backtest.StatusPayload{ - RunID: meta.RunID, - State: meta.State, - ProgressPct: meta.Summary.ProgressPct, - ProcessedBars: meta.Summary.ProcessedBars, - CurrentTime: 0, - DecisionCycle: meta.Summary.ProcessedBars, - Equity: meta.Summary.EquityLast, - UnrealizedPnL: 0, - RealizedPnL: 0, - Note: meta.Summary.LiquidationNote, - LastUpdatedIso: meta.UpdatedAt.Format(time.RFC3339), - } - c.JSON(http.StatusOK, payload) -} - -func (s *Server) handleBacktestRuns(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - rawUserID := strings.TrimSpace(c.GetString("user_id")) - userID := normalizeUserID(rawUserID) - filterByUser := rawUserID != "" && rawUserID != "admin" - - metas, err := s.backtestManager.ListRuns() - if err != nil { - SafeInternalError(c, "List backtest runs", err) - return - } - stateFilter := strings.ToLower(strings.TrimSpace(c.Query("state"))) - search := strings.ToLower(strings.TrimSpace(c.Query("search"))) - limit := queryInt(c, "limit", 50) - offset := queryInt(c, "offset", 0) - if limit <= 0 { - limit = 50 - } - if offset < 0 { - offset = 0 - } - - filtered := make([]*backtest.RunMetadata, 0, len(metas)) - for _, meta := range metas { - if stateFilter != "" && !strings.EqualFold(string(meta.State), stateFilter) { - continue - } - if search != "" { - target := strings.ToLower(meta.RunID + " " + meta.Summary.DecisionTF + " " + meta.Label + " " + meta.LastError) - if !strings.Contains(target, search) { - continue - } - } - if filterByUser { - owner := strings.TrimSpace(meta.UserID) - if owner != "" && owner != userID { - continue - } - } - filtered = append(filtered, meta) - } - - total := len(filtered) - start := offset - if start > total { - start = total - } - end := offset + limit - if end > total { - end = total - } - page := filtered[start:end] - - c.JSON(http.StatusOK, gin.H{ - "total": total, - "items": page, - }) -} - -func (s *Server) handleBacktestEquity(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - - userID := normalizeUserID(c.GetString("user_id")) - - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - timeframe := c.Query("tf") - limit := queryInt(c, "limit", 1000) - - points, err := s.backtestManager.LoadEquity(runID, timeframe, limit) - if err != nil { - SafeError(c, http.StatusBadRequest, "Failed to load equity data", err) - return - } - c.JSON(http.StatusOK, points) -} - -func (s *Server) handleBacktestTrades(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - - userID := normalizeUserID(c.GetString("user_id")) - - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - limit := queryInt(c, "limit", 1000) - - events, err := s.backtestManager.LoadTrades(runID, limit) - if err != nil { - SafeError(c, http.StatusBadRequest, "Failed to load trades", err) - return - } - c.JSON(http.StatusOK, events) -} - -func (s *Server) handleBacktestMetrics(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - - userID := normalizeUserID(c.GetString("user_id")) - - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - - metrics, err := s.backtestManager.GetMetrics(runID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) || errors.Is(err, os.ErrNotExist) { - c.JSON(http.StatusAccepted, gin.H{"error": "metrics not ready yet"}) - return - } - SafeError(c, http.StatusBadRequest, "Failed to load metrics", err) - return - } - c.JSON(http.StatusOK, metrics) -} - -func (s *Server) handleBacktestTrace(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - userID := normalizeUserID(c.GetString("user_id")) - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - cycle := queryInt(c, "cycle", 0) - record, err := s.backtestManager.GetTrace(runID, cycle) - if err != nil { - SafeNotFound(c, "Trace record") - return - } - c.JSON(http.StatusOK, record) -} - -func (s *Server) handleBacktestDecisions(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - userID := normalizeUserID(c.GetString("user_id")) - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - limit := queryInt(c, "limit", 20) - offset := queryInt(c, "offset", 0) - if limit <= 0 { - limit = 20 - } - if limit > 200 { - limit = 200 - } - if offset < 0 { - offset = 0 - } - - records, err := backtest.LoadDecisionRecords(runID, limit, offset) - if err != nil { - SafeInternalError(c, "Load decision records", err) - return - } - c.JSON(http.StatusOK, records) -} - -func (s *Server) handleBacktestExport(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - userID := normalizeUserID(c.GetString("user_id")) - runID := c.Query("run_id") - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if _, err := s.ensureBacktestRunOwnership(runID, userID); writeBacktestAccessError(c, err) { - return - } - path, err := s.backtestManager.ExportRun(runID) - if err != nil { - SafeError(c, http.StatusBadRequest, "Failed to export backtest", err) - return - } - defer os.Remove(path) - filename := fmt.Sprintf("%s_export.zip", runID) - c.FileAttachment(path, filename) -} - -func (s *Server) handleBacktestKlines(c *gin.Context) { - if s.backtestManager == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "backtest manager unavailable"}) - return - } - userID := normalizeUserID(c.GetString("user_id")) - runID := c.Query("run_id") - symbol := c.Query("symbol") - timeframe := c.Query("timeframe") - - if runID == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "run_id is required"}) - return - } - if symbol == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "symbol is required"}) - return - } - - meta, err := s.ensureBacktestRunOwnership(runID, userID) - if writeBacktestAccessError(c, err) { - return - } - - // Load config to get time range - cfg, err := backtest.LoadConfig(runID) - if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "failed to load backtest config"}) - return - } - - // Use decision timeframe if not specified - if timeframe == "" { - timeframe = cfg.DecisionTimeframe - if timeframe == "" { - timeframe = "15m" - } - } - - // Fetch klines for the backtest time range - startTime := time.Unix(cfg.StartTS, 0) - endTime := time.Unix(cfg.EndTS, 0) - - klines, err := market.GetKlinesRange(symbol, timeframe, startTime, endTime) - if err != nil { - SafeInternalError(c, "Fetch klines", err) - return - } - - // Convert to response format - type KlineResponse struct { - Time int64 `json:"time"` - Open float64 `json:"open"` - High float64 `json:"high"` - Low float64 `json:"low"` - Close float64 `json:"close"` - Volume float64 `json:"volume"` - } - - result := make([]KlineResponse, len(klines)) - for i, k := range klines { - result[i] = KlineResponse{ - Time: k.OpenTime / 1000, // Convert to seconds for lightweight-charts - Open: k.Open, - High: k.High, - Low: k.Low, - Close: k.Close, - Volume: k.Volume, - } - } - - c.JSON(http.StatusOK, gin.H{ - "symbol": symbol, - "timeframe": timeframe, - "start_ts": cfg.StartTS, - "end_ts": cfg.EndTS, - "count": len(result), - "klines": result, - "run_id": meta.RunID, - }) -} - -func queryInt(c *gin.Context, name string, fallback int) int { - if value := c.Query(name); value != "" { - if v, err := strconv.Atoi(value); err == nil { - return v - } - } - return fallback -} - -var errBacktestForbidden = errors.New("backtest run forbidden") - -func normalizeUserID(id string) string { - id = strings.TrimSpace(id) - if id == "" { - return "default" - } - return id -} - -func (s *Server) ensureBacktestRunOwnership(runID, userID string) (*backtest.RunMetadata, error) { - if s.backtestManager == nil { - return nil, fmt.Errorf("backtest manager unavailable") - } - meta, err := s.backtestManager.LoadMetadata(runID) - if err != nil { - return nil, err - } - if userID == "" || userID == "admin" { - return meta, nil - } - owner := strings.TrimSpace(meta.UserID) - if owner == "" { - return meta, nil - } - if owner != userID { - return nil, errBacktestForbidden - } - return meta, nil -} - -func writeBacktestAccessError(c *gin.Context, err error) bool { - if err == nil { - return false - } - switch { - case errors.Is(err, errBacktestForbidden): - SafeForbidden(c, "No permission to access this backtest task") - case errors.Is(err, os.ErrNotExist), errors.Is(err, sql.ErrNoRows): - SafeNotFound(c, "Backtest task") - default: - SafeInternalError(c, "Access backtest", err) - } - return true -} - -// resolveStrategyCoins fetches coins based on strategy's coin source configuration -func (s *Server) resolveStrategyCoins(strategyConfig *store.StrategyConfig) ([]string, error) { - if strategyConfig == nil { - return nil, fmt.Errorf("strategy config is nil") - } - - coinSource := strategyConfig.CoinSource - var symbols []string - symbolSet := make(map[string]bool) - - // Handle empty source_type - check flags for backward compatibility - sourceType := coinSource.SourceType - if sourceType == "" { - if coinSource.UseAI500 && coinSource.UseOITop { - sourceType = "mixed" - } else if coinSource.UseAI500 { - sourceType = "ai500" - } else if coinSource.UseOITop { - sourceType = "oi_top" - } else if len(coinSource.StaticCoins) > 0 { - sourceType = "static" - } else { - return nil, fmt.Errorf("strategy has no coin source configured") - } - logger.Infof("📊 Inferred source_type=%s from flags", sourceType) - } - - switch sourceType { - case "static": - for _, sym := range coinSource.StaticCoins { - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - - case "ai500": - limit := coinSource.AI500Limit - if limit <= 0 { - limit = 30 - } - logger.Infof("📊 Fetching AI500 coins with limit=%d", limit) - coins, err := nofxos.DefaultClient().GetTopRatedCoins(limit) - if err != nil { - return nil, fmt.Errorf("failed to get AI500 coins: %w", err) - } - logger.Infof("📊 Got %d coins from AI500: %v", len(coins), coins) - for _, sym := range coins { - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - - case "oi_top": - coins, err := nofxos.DefaultClient().GetOITopSymbols() - if err != nil { - return nil, fmt.Errorf("failed to get OI Top coins: %w", err) - } - limit := coinSource.OITopLimit - if limit <= 0 || limit > len(coins) { - limit = len(coins) - } - for i, sym := range coins { - if i >= limit { - break - } - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - - case "mixed": - // Get from AI500 - if coinSource.UseAI500 { - limit := coinSource.AI500Limit - if limit <= 0 { - limit = 30 - } - coins, err := nofxos.DefaultClient().GetTopRatedCoins(limit) - if err != nil { - logger.Warnf("Failed to get AI500 coins: %v", err) - } else { - for _, sym := range coins { - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - } - } - - // Get from OI Top - if coinSource.UseOITop { - coins, err := nofxos.DefaultClient().GetOITopSymbols() - if err != nil { - logger.Warnf("Failed to get OI Top coins: %v", err) - } else { - limit := coinSource.OITopLimit - if limit <= 0 || limit > len(coins) { - limit = len(coins) - } - for i, sym := range coins { - if i >= limit { - break - } - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - } - } - - // Add static coins - for _, sym := range coinSource.StaticCoins { - sym = market.Normalize(sym) - if !symbolSet[sym] { - symbols = append(symbols, sym) - symbolSet[sym] = true - } - } - - default: - return nil, fmt.Errorf("unknown coin source type: %s", sourceType) - } - - if len(symbols) == 0 { - return nil, fmt.Errorf("no coins resolved from strategy") - } - - logger.Infof("📊 Final resolved symbols: %d coins - %v", len(symbols), symbols) - return symbols, nil -} - -func (s *Server) resolveBacktestAIConfig(cfg *backtest.BacktestConfig, userID string) error { - if cfg == nil { - return fmt.Errorf("config is nil") - } - if s.store == nil { - return fmt.Errorf("System database not ready, cannot load AI model configuration") - } - - cfg.UserID = normalizeUserID(userID) - - return s.hydrateBacktestAIConfig(cfg) -} - -func (s *Server) hydrateBacktestAIConfig(cfg *backtest.BacktestConfig) error { - if cfg == nil { - return fmt.Errorf("config is nil") - } - if s.store == nil { - return fmt.Errorf("System database not ready, cannot load AI model configuration") - } - - cfg.UserID = normalizeUserID(cfg.UserID) - modelID := strings.TrimSpace(cfg.AIModelID) - - var ( - model *store.AIModel - err error - ) - - if modelID != "" { - model, err = s.store.AIModel().Get(cfg.UserID, modelID) - if err != nil { - return fmt.Errorf("Failed to load AI model: %w", err) - } - } else { - model, err = s.store.AIModel().GetDefault(cfg.UserID) - if err != nil { - return fmt.Errorf("No available AI model found: %w", err) - } - cfg.AIModelID = model.ID - } - - if !model.Enabled { - return fmt.Errorf("AI model %s is not enabled yet", model.Name) - } - - apiKey := strings.TrimSpace(string(model.APIKey)) - if apiKey == "" { - return fmt.Errorf("AI model %s is missing API Key, please configure it in the system first", model.Name) - } - - provider := strings.ToLower(strings.TrimSpace(model.Provider)) - // Ensure provider is never empty or "inherit" - infer from model name if needed - if provider == "" || provider == "inherit" { - modelNameLower := strings.ToLower(model.Name) - if strings.Contains(modelNameLower, "claude") || strings.Contains(modelNameLower, "anthropic") { - provider = "anthropic" - } else if strings.Contains(modelNameLower, "gpt") || strings.Contains(modelNameLower, "openai") { - provider = "openai" - } else if strings.Contains(modelNameLower, "gemini") || strings.Contains(modelNameLower, "google") { - provider = "google" - } else if strings.Contains(modelNameLower, "deepseek") { - provider = "deepseek" - } else if strings.Contains(modelNameLower, "minimax") { - provider = "minimax" - } else if model.CustomAPIURL != "" { - provider = "custom" - } else { - provider = "openai" // default fallback - } - logger.Infof("📊 Inferred AI provider '%s' from model name '%s'", provider, model.Name) - } - cfg.AICfg.Provider = provider - cfg.AICfg.APIKey = apiKey - cfg.AICfg.BaseURL = strings.TrimSpace(model.CustomAPIURL) - modelName := strings.TrimSpace(model.CustomModelName) - if cfg.AICfg.Model == "" { - cfg.AICfg.Model = modelName - } - cfg.AICfg.Model = strings.TrimSpace(cfg.AICfg.Model) - - if cfg.AICfg.Provider == "custom" { - if cfg.AICfg.BaseURL == "" { - return fmt.Errorf("Custom AI model requires API URL configuration") - } - if cfg.AICfg.Model == "" { - return fmt.Errorf("Custom AI model requires model name configuration") - } - } - - return nil -} diff --git a/api/server.go b/api/server.go index 55944022..cfe8be2d 100644 --- a/api/server.go +++ b/api/server.go @@ -6,7 +6,6 @@ import ( "net" "net/http" "nofx/auth" - "nofx/backtest" "nofx/crypto" "nofx/logger" "nofx/manager" @@ -23,14 +22,13 @@ type Server struct { traderManager *manager.TraderManager store *store.Store cryptoHandler *CryptoHandler - backtestManager *backtest.Manager httpServer *http.Server port int telegramReloadCh chan<- struct{} // signal Telegram bot to reload } // NewServer Creates API server -func NewServer(traderManager *manager.TraderManager, st *store.Store, cryptoService *crypto.CryptoService, backtestManager *backtest.Manager, port int) *Server { +func NewServer(traderManager *manager.TraderManager, st *store.Store, cryptoService *crypto.CryptoService, port int) *Server { // Set to Release mode (reduce log output) gin.SetMode(gin.ReleaseMode) @@ -43,12 +41,11 @@ func NewServer(traderManager *manager.TraderManager, st *store.Store, cryptoServ cryptoHandler := NewCryptoHandler(cryptoService) s := &Server{ - router: router, - traderManager: traderManager, - store: st, - cryptoHandler: cryptoHandler, - backtestManager: backtestManager, - port: port, + router: router, + traderManager: traderManager, + store: st, + cryptoHandler: cryptoHandler, + port: port, } // Setup routes @@ -342,9 +339,6 @@ Returns the most recent AI decision for each symbol analyzed in the last scan cy Returns: {"total_trades":,"winning_trades":,"win_rate":,"total_pnl":,"sharpe_ratio":,"max_drawdown":}`, s.handleStatistics) - // Backtest routes - backtest := protected.Group("/backtest") - s.registerBacktestRoutes(backtest) } } } diff --git a/backtest/account.go b/backtest/account.go deleted file mode 100644 index 49468c1a..00000000 --- a/backtest/account.go +++ /dev/null @@ -1,267 +0,0 @@ -package backtest - -import ( - "fmt" - "math" - "strings" -) - -const epsilon = 1e-8 - -type position struct { - Symbol string - Side string - Quantity float64 - EntryPrice float64 - Leverage int - Margin float64 - Notional float64 - LiquidationPrice float64 - OpenTime int64 - AccumulatedFee float64 // Total fees paid (opening + any additions) -} - -type BacktestAccount struct { - initialBalance float64 - cash float64 - feeRate float64 - slippageRate float64 - positions map[string]*position - realizedPnL float64 -} - -func NewBacktestAccount(initialBalance, feeBps, slippageBps float64) *BacktestAccount { - return &BacktestAccount{ - initialBalance: initialBalance, - cash: initialBalance, - feeRate: feeBps / 10000.0, - slippageRate: slippageBps / 10000.0, - positions: make(map[string]*position), - } -} - -func positionKey(symbol, side string) string { - return strings.ToUpper(symbol) + ":" + side -} - -func (acc *BacktestAccount) ensurePosition(symbol, side string) *position { - key := positionKey(symbol, side) - if pos, ok := acc.positions[key]; ok { - return pos - } - pos := &position{Symbol: strings.ToUpper(symbol), Side: side} - acc.positions[key] = pos - return pos -} - -func (acc *BacktestAccount) removePosition(pos *position) { - key := positionKey(pos.Symbol, pos.Side) - delete(acc.positions, key) -} - -func (acc *BacktestAccount) Open(symbol, side string, quantity float64, leverage int, price float64, ts int64) (*position, float64, float64, error) { - if quantity <= 0 { - return nil, 0, 0, fmt.Errorf("quantity must be positive") - } - if leverage <= 0 { - return nil, 0, 0, fmt.Errorf("leverage must be positive") - } - - execPrice := applySlippage(price, acc.slippageRate, side, true) - notional := execPrice * quantity - margin := notional / float64(leverage) - fee := notional * acc.feeRate - - if margin+fee > acc.cash+epsilon { - return nil, 0, 0, fmt.Errorf("insufficient cash: need %.2f", margin+fee) - } - - acc.cash -= margin + fee - - pos := acc.ensurePosition(symbol, side) - - if pos.Quantity < epsilon { - pos.Quantity = quantity - pos.EntryPrice = execPrice - pos.Leverage = leverage - pos.Margin = margin - pos.Notional = notional - pos.OpenTime = ts - pos.LiquidationPrice = computeLiquidation(execPrice, leverage, side) - pos.AccumulatedFee = fee // Track opening fee - } else { - if leverage != pos.Leverage { - // Use weighted average leverage (approximate) - weightedMargin := pos.Margin + margin - pos.Leverage = int(math.Round((pos.Notional + notional) / weightedMargin)) - } - pos.Notional += notional - pos.Margin += margin - pos.EntryPrice = ((pos.EntryPrice * pos.Quantity) + execPrice*quantity) / (pos.Quantity + quantity) - pos.Quantity += quantity - pos.LiquidationPrice = computeLiquidation(pos.EntryPrice, pos.Leverage, side) - pos.AccumulatedFee += fee // Add to accumulated fee for position additions - } - - return pos, fee, execPrice, nil -} - -func (acc *BacktestAccount) Close(symbol, side string, quantity float64, price float64) (float64, float64, float64, error) { - key := positionKey(symbol, side) - pos, ok := acc.positions[key] - if !ok || pos.Quantity <= epsilon { - return 0, 0, 0, fmt.Errorf("no active %s position for %s", side, symbol) - } - - if quantity <= 0 || quantity > pos.Quantity+epsilon { - if math.Abs(quantity) <= epsilon { - quantity = pos.Quantity - } else { - return 0, 0, 0, fmt.Errorf("invalid close quantity") - } - } - - execPrice := applySlippage(price, acc.slippageRate, side, false) - closeNotional := execPrice * quantity // Notional at close price (for fee calculation) - closingFee := closeNotional * acc.feeRate - - // Calculate proportional values based on the portion being closed - closePortion := quantity / pos.Quantity - openingFeePortion := pos.AccumulatedFee * closePortion - totalFee := closingFee + openingFeePortion - - realized := realizedPnL(pos, quantity, execPrice) - - marginPortion := pos.Margin * closePortion - // BUG FIX: Calculate notional portion based on ENTRY price, not close price - // pos.Notional tracks the total notional at entry, so we must subtract proportionally - entryNotionalPortion := pos.Notional * closePortion - - // Note: Opening fee was already deducted from cash when opening, so we only deduct closing fee here - acc.cash += marginPortion + realized - closingFee - // But for realized P&L tracking, we include both fees - acc.realizedPnL += realized - totalFee - - pos.Quantity -= quantity - pos.Notional -= entryNotionalPortion // FIX: Use entry notional portion, not close notional - pos.Margin -= marginPortion - pos.AccumulatedFee -= openingFeePortion // Reduce tracked opening fee - - if pos.Quantity <= epsilon { - acc.removePosition(pos) - } - - // Return total fee (opening + closing) so caller can calculate accurate P&L - return realized, totalFee, execPrice, nil -} - -func (acc *BacktestAccount) TotalEquity(priceMap map[string]float64) (float64, float64, map[string]float64) { - unrealized := 0.0 - margin := 0.0 - perSymbol := make(map[string]float64) - for _, pos := range acc.positions { - price := priceMap[pos.Symbol] - pnl := unrealizedPnL(pos, price) - unrealized += pnl - margin += pos.Margin - perSymbol[pos.Symbol+":"+pos.Side] = pnl - } - return acc.cash + margin + unrealized, unrealized, perSymbol -} - -func applySlippage(price float64, rate float64, side string, isOpen bool) float64 { - if rate <= 0 { - return price - } - adjust := 1.0 - if side == "long" { - if isOpen { - adjust += rate - } else { - adjust -= rate - } - } else { - if isOpen { - adjust -= rate - } else { - adjust += rate - } - } - return price * adjust -} - -func computeLiquidation(entry float64, leverage int, side string) float64 { - if leverage <= 0 { - return 0 - } - lev := float64(leverage) - if side == "long" { - return entry * (1.0 - 1.0/lev) - } - return entry * (1.0 + 1.0/lev) -} - -func realizedPnL(pos *position, qty, price float64) float64 { - if pos.Side == "long" { - return (price - pos.EntryPrice) * qty - } - return (pos.EntryPrice - price) * qty -} - -func unrealizedPnL(pos *position, price float64) float64 { - if pos.Side == "long" { - return (price - pos.EntryPrice) * pos.Quantity - } - return (pos.EntryPrice - price) * pos.Quantity -} - -func (acc *BacktestAccount) Positions() []*position { - list := make([]*position, 0, len(acc.positions)) - for _, pos := range acc.positions { - list = append(list, pos) - } - return list -} - -func (acc *BacktestAccount) positionLeverage(symbol, side string) int { - key := positionKey(symbol, side) - if pos, ok := acc.positions[key]; ok && pos.Quantity > epsilon { - return pos.Leverage - } - return 0 -} - -func (acc *BacktestAccount) Cash() float64 { - return acc.cash -} - -func (acc *BacktestAccount) InitialBalance() float64 { - return acc.initialBalance -} - -func (acc *BacktestAccount) RealizedPnL() float64 { - return acc.realizedPnL -} - -// RestoreFromSnapshots restores account state from checkpoint. -func (acc *BacktestAccount) RestoreFromSnapshots(cash float64, realized float64, snaps []PositionSnapshot) { - acc.cash = cash - acc.realizedPnL = realized - acc.positions = make(map[string]*position) - for _, snap := range snaps { - pos := &position{ - Symbol: snap.Symbol, - Side: snap.Side, - Quantity: snap.Quantity, - EntryPrice: snap.AvgPrice, - Leverage: snap.Leverage, - Margin: snap.MarginUsed, - Notional: snap.Quantity * snap.AvgPrice, - LiquidationPrice: snap.LiquidationPrice, - OpenTime: snap.OpenTime, - AccumulatedFee: snap.AccumulatedFee, - } - key := positionKey(pos.Symbol, pos.Side) - acc.positions[key] = pos - } -} diff --git a/backtest/ai_client.go b/backtest/ai_client.go deleted file mode 100644 index 5395b430..00000000 --- a/backtest/ai_client.go +++ /dev/null @@ -1,72 +0,0 @@ -package backtest - -import ( - "fmt" - "strings" - - "nofx/mcp" - _ "nofx/mcp/payment" - _ "nofx/mcp/provider" -) - -// configureMCPClient creates/clones an MCP client based on configuration (returns mcp.AIClient interface). -// Note: mcp.New() returns an interface type; here we convert to concrete implementation before copying to avoid concurrent shared state. -func configureMCPClient(cfg BacktestConfig, base mcp.AIClient) (mcp.AIClient, error) { - providerName := strings.ToLower(strings.TrimSpace(cfg.AICfg.Provider)) - - // Inherit base client - if providerName == "" || providerName == "inherit" || providerName == "default" { - client := cloneBaseClient(base) - if cfg.AICfg.APIKey != "" || cfg.AICfg.BaseURL != "" || cfg.AICfg.Model != "" { - client.SetAPIKey(cfg.AICfg.APIKey, cfg.AICfg.BaseURL, cfg.AICfg.Model) - } - return client, nil - } - - // Custom provider uses cloned base - if providerName == "custom" { - if cfg.AICfg.BaseURL == "" || cfg.AICfg.APIKey == "" || cfg.AICfg.Model == "" { - return nil, fmt.Errorf("custom provider requires base_url, api key and model") - } - client := cloneBaseClient(base) - client.SetAPIKey(cfg.AICfg.APIKey, cfg.AICfg.BaseURL, cfg.AICfg.Model) - return client, nil - } - - // Create client via registry - client := mcp.NewAIClientByProvider(providerName) - if client == nil { - return nil, fmt.Errorf("unsupported ai provider %s", cfg.AICfg.Provider) - } - - if cfg.AICfg.APIKey == "" { - return nil, fmt.Errorf("%s provider requires api key", providerName) - } - - // Payment providers ignore custom URL - switch providerName { - case "blockrun-base", "blockrun-sol", "claw402": - client.SetAPIKey(cfg.AICfg.APIKey, "", cfg.AICfg.Model) - default: - client.SetAPIKey(cfg.AICfg.APIKey, cfg.AICfg.BaseURL, cfg.AICfg.Model) - } - return client, nil -} - -// cloneBaseClient copies the base client to avoid shared mutable state. -// Uses the ClientEmbedder interface to extract the underlying *mcp.Client -// from any provider type that embeds it. -func cloneBaseClient(base mcp.AIClient) *mcp.Client { - if embedder, ok := base.(mcp.ClientEmbedder); ok { - if inner := embedder.BaseClient(); inner != nil { - cp := *inner - return &cp - } - } - if c, ok := base.(*mcp.Client); ok { - cp := *c - return &cp - } - // Fall back to a new default client - return mcp.NewClient().(*mcp.Client) -} diff --git a/backtest/aicache.go b/backtest/aicache.go deleted file mode 100644 index f5627e92..00000000 --- a/backtest/aicache.go +++ /dev/null @@ -1,168 +0,0 @@ -package backtest - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "os" - "path/filepath" - "sync" - - "nofx/kernel" - "nofx/market" -) - -type cachedDecision struct { - Key string `json:"key"` - PromptVariant string `json:"prompt_variant"` - Timestamp int64 `json:"ts"` - Decision *kernel.FullDecision `json:"decision"` -} - -// AICache persists AI decisions for repeated backtesting or replay. -type AICache struct { - mu sync.RWMutex - path string - Entries map[string]cachedDecision `json:"entries"` -} - -func LoadAICache(path string) (*AICache, error) { - if path == "" { - return nil, fmt.Errorf("ai cache path is empty") - } - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - - cache := &AICache{ - path: path, - Entries: make(map[string]cachedDecision), - } - - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return cache, nil - } - return nil, err - } - if len(data) == 0 { - return cache, nil - } - if err := json.Unmarshal(data, cache); err != nil { - return nil, err - } - if cache.Entries == nil { - cache.Entries = make(map[string]cachedDecision) - } - return cache, nil -} - -func (c *AICache) Path() string { - if c == nil { - return "" - } - return c.path -} - -func (c *AICache) Get(key string) (*kernel.FullDecision, bool) { - if c == nil || key == "" { - return nil, false - } - c.mu.RLock() - entry, ok := c.Entries[key] - c.mu.RUnlock() - if !ok || entry.Decision == nil { - return nil, false - } - return cloneDecision(entry.Decision), true -} - -func (c *AICache) Put(key string, variant string, ts int64, decision *kernel.FullDecision) error { - if c == nil || key == "" || decision == nil { - return nil - } - entry := cachedDecision{ - Key: key, - PromptVariant: variant, - Timestamp: ts, - Decision: cloneDecision(decision), - } - c.mu.Lock() - c.Entries[key] = entry - c.mu.Unlock() - return c.save() -} - -func (c *AICache) save() error { - if c == nil || c.path == "" { - return nil - } - c.mu.RLock() - data, err := json.MarshalIndent(c, "", " ") - c.mu.RUnlock() - if err != nil { - return err - } - return writeFileAtomic(c.path, data, 0o644) -} - -func cloneDecision(src *kernel.FullDecision) *kernel.FullDecision { - if src == nil { - return nil - } - data, err := json.Marshal(src) - if err != nil { - return nil - } - var dst kernel.FullDecision - if err := json.Unmarshal(data, &dst); err != nil { - return nil - } - return &dst -} - -func computeCacheKey(ctx *kernel.Context, variant string, ts int64) (string, error) { - if ctx == nil { - return "", fmt.Errorf("context is nil") - } - payload := struct { - Variant string `json:"variant"` - Timestamp int64 `json:"ts"` - CurrentTime string `json:"current_time"` - Account kernel.AccountInfo `json:"account"` - Positions []kernel.PositionInfo `json:"positions"` - CandidateCoins []kernel.CandidateCoin `json:"candidate_coins"` - MarketData map[string]market.Data `json:"market"` - MarginUsedPct float64 `json:"margin_used_pct"` - Runtime int `json:"runtime_minutes"` - CallCount int `json:"call_count"` - }{ - Variant: variant, - Timestamp: ts, - CurrentTime: ctx.CurrentTime, - Account: ctx.Account, - Positions: ctx.Positions, - CandidateCoins: ctx.CandidateCoins, - MarginUsedPct: ctx.Account.MarginUsedPct, - Runtime: ctx.RuntimeMinutes, - CallCount: ctx.CallCount, - MarketData: make(map[string]market.Data, len(ctx.MarketDataMap)), - } - - for symbol, data := range ctx.MarketDataMap { - if data == nil { - continue - } - payload.MarketData[symbol] = *data - } - - bytes, err := json.Marshal(payload) - if err != nil { - return "", err - } - sum := sha256.Sum256(bytes) - return hex.EncodeToString(sum[:]), nil -} diff --git a/backtest/config.go b/backtest/config.go deleted file mode 100644 index 629fac97..00000000 --- a/backtest/config.go +++ /dev/null @@ -1,285 +0,0 @@ -package backtest - -import ( - "fmt" - "strings" - "time" - - "nofx/market" - "nofx/store" -) - -// AIConfig defines the AI client configuration used in backtesting. -type AIConfig struct { - Provider string `json:"provider"` - Model string `json:"model"` - APIKey string `json:"key"` - SecretKey string `json:"secret_key,omitempty"` - BaseURL string `json:"base_url,omitempty"` - Temperature float64 `json:"temperature,omitempty"` -} - -type LeverageConfig struct { - BTCETHLeverage int `json:"btc_eth_leverage"` - AltcoinLeverage int `json:"altcoin_leverage"` -} - -// BacktestConfig describes the input configuration for a backtest run. -type BacktestConfig struct { - RunID string `json:"run_id"` - UserID string `json:"user_id,omitempty"` - AIModelID string `json:"ai_model_id,omitempty"` - StrategyID string `json:"strategy_id,omitempty"` // Optional: use saved strategy from Strategy Studio - Symbols []string `json:"symbols"` - Timeframes []string `json:"timeframes"` - DecisionTimeframe string `json:"decision_timeframe"` - DecisionCadenceNBars int `json:"decision_cadence_nbars"` - StartTS int64 `json:"start_ts"` - EndTS int64 `json:"end_ts"` - InitialBalance float64 `json:"initial_balance"` - FeeBps float64 `json:"fee_bps"` - SlippageBps float64 `json:"slippage_bps"` - FillPolicy string `json:"fill_policy"` - PromptVariant string `json:"prompt_variant"` - PromptTemplate string `json:"prompt_template"` - CustomPrompt string `json:"custom_prompt"` - OverrideBasePrompt bool `json:"override_prompt"` - CacheAI bool `json:"cache_ai"` - ReplayOnly bool `json:"replay_only"` - - AICfg AIConfig `json:"ai"` - Leverage LeverageConfig `json:"leverage"` - - SharedAICachePath string `json:"ai_cache_path,omitempty"` - CheckpointIntervalBars int `json:"checkpoint_interval_bars,omitempty"` - CheckpointIntervalSeconds int `json:"checkpoint_interval_seconds,omitempty"` - ReplayDecisionDir string `json:"replay_decision_dir,omitempty"` - - // Internal: loaded strategy config (set by Manager when StrategyID is provided) - loadedStrategy *store.StrategyConfig `json:"-"` -} - -// Validate performs validity checks on the configuration and fills in default values. -func (cfg *BacktestConfig) Validate() error { - if cfg == nil { - return fmt.Errorf("config is nil") - } - cfg.RunID = strings.TrimSpace(cfg.RunID) - if cfg.RunID == "" { - return fmt.Errorf("run_id cannot be empty") - } - cfg.UserID = strings.TrimSpace(cfg.UserID) - if cfg.UserID == "" { - cfg.UserID = "default" - } - cfg.AIModelID = strings.TrimSpace(cfg.AIModelID) - - if len(cfg.Symbols) == 0 { - return fmt.Errorf("at least one symbol is required") - } - for i, sym := range cfg.Symbols { - cfg.Symbols[i] = market.Normalize(sym) - } - - if len(cfg.Timeframes) == 0 { - cfg.Timeframes = []string{"3m", "15m", "4h"} - } - normTF := make([]string, 0, len(cfg.Timeframes)) - for _, tf := range cfg.Timeframes { - normalized, err := market.NormalizeTimeframe(tf) - if err != nil { - return fmt.Errorf("invalid timeframe '%s': %w", tf, err) - } - normTF = append(normTF, normalized) - } - cfg.Timeframes = normTF - - if cfg.DecisionTimeframe == "" { - cfg.DecisionTimeframe = cfg.Timeframes[0] - } - normalizedDecision, err := market.NormalizeTimeframe(cfg.DecisionTimeframe) - if err != nil { - return fmt.Errorf("invalid decision_timeframe: %w", err) - } - cfg.DecisionTimeframe = normalizedDecision - - if cfg.DecisionCadenceNBars <= 0 { - cfg.DecisionCadenceNBars = 20 - } - - if cfg.StartTS <= 0 || cfg.EndTS <= 0 || cfg.EndTS <= cfg.StartTS { - return fmt.Errorf("invalid start_ts/end_ts") - } - - if cfg.InitialBalance <= 0 { - cfg.InitialBalance = 1000 - } - - if cfg.FillPolicy == "" { - cfg.FillPolicy = FillPolicyNextOpen - } - if err := validateFillPolicy(cfg.FillPolicy); err != nil { - return err - } - - if cfg.CheckpointIntervalBars <= 0 { - cfg.CheckpointIntervalBars = 20 - } - if cfg.CheckpointIntervalSeconds <= 0 { - cfg.CheckpointIntervalSeconds = 2 - } - - cfg.PromptVariant = strings.TrimSpace(cfg.PromptVariant) - if cfg.PromptVariant == "" { - cfg.PromptVariant = "baseline" - } - cfg.PromptTemplate = strings.TrimSpace(cfg.PromptTemplate) - if cfg.PromptTemplate == "" { - cfg.PromptTemplate = "default" - } - cfg.CustomPrompt = strings.TrimSpace(cfg.CustomPrompt) - - if cfg.AICfg.Provider == "" { - cfg.AICfg.Provider = "inherit" - } - if cfg.AICfg.Temperature == 0 { - cfg.AICfg.Temperature = 0.4 - } - - if cfg.Leverage.BTCETHLeverage <= 0 { - cfg.Leverage.BTCETHLeverage = 5 - } - if cfg.Leverage.AltcoinLeverage <= 0 { - cfg.Leverage.AltcoinLeverage = 5 - } - - return nil -} - -// Duration returns the backtest interval duration. -func (cfg *BacktestConfig) Duration() time.Duration { - if cfg == nil { - return 0 - } - return time.Unix(cfg.EndTS, 0).Sub(time.Unix(cfg.StartTS, 0)) -} - -const ( - // FillPolicyNextOpen uses the open price of the next bar for execution. - FillPolicyNextOpen = "next_open" - // FillPolicyBarVWAP uses the approximate VWAP of the current bar for execution. - FillPolicyBarVWAP = "bar_vwap" - // FillPolicyMidPrice uses the mid-price (high+low)/2 for execution. - FillPolicyMidPrice = "mid" -) - -func validateFillPolicy(policy string) error { - switch policy { - case FillPolicyNextOpen, FillPolicyBarVWAP, FillPolicyMidPrice: - return nil - default: - return fmt.Errorf("unsupported fill_policy '%s'", policy) - } -} - -// SetLoadedStrategy sets the loaded strategy config from database. -func (cfg *BacktestConfig) SetLoadedStrategy(strategy *store.StrategyConfig) { - cfg.loadedStrategy = strategy -} - -// ToStrategyConfig converts BacktestConfig to StrategyConfig for unified prompt generation. -// This ensures backtest uses the same StrategyEngine logic as live trading. -// If a strategy was loaded from database (via StrategyID), it will be used with overrides. -func (cfg *BacktestConfig) ToStrategyConfig() *store.StrategyConfig { - // If a strategy was loaded from database, use it with some overrides - if cfg.loadedStrategy != nil { - result := *cfg.loadedStrategy // Make a copy - - // Override coin source with backtest symbols (backtest-specified pairs take priority) - if len(cfg.Symbols) > 0 { - result.CoinSource.SourceType = "static" - result.CoinSource.StaticCoins = cfg.Symbols - result.CoinSource.UseAI500 = false - result.CoinSource.UseOITop = false - } - - // Override timeframes with backtest config - if len(cfg.Timeframes) > 0 { - result.Indicators.Klines.SelectedTimeframes = cfg.Timeframes - result.Indicators.Klines.PrimaryTimeframe = cfg.Timeframes[0] - if len(cfg.Timeframes) > 1 { - result.Indicators.Klines.LongerTimeframe = cfg.Timeframes[len(cfg.Timeframes)-1] - } - result.Indicators.Klines.EnableMultiTimeframe = len(cfg.Timeframes) > 1 - } - - // Override leverage with backtest config - if cfg.Leverage.BTCETHLeverage > 0 { - result.RiskControl.BTCETHMaxLeverage = cfg.Leverage.BTCETHLeverage - } - if cfg.Leverage.AltcoinLeverage > 0 { - result.RiskControl.AltcoinMaxLeverage = cfg.Leverage.AltcoinLeverage - } - - // Override custom prompt if provided in backtest config - if cfg.CustomPrompt != "" { - result.CustomPrompt = cfg.CustomPrompt - } - - return &result - } - - // Fallback: build strategy config from backtest config (original logic) - primaryTF := "5m" - longerTF := "4h" - if len(cfg.Timeframes) > 0 { - primaryTF = cfg.Timeframes[0] - } - if len(cfg.Timeframes) > 1 { - longerTF = cfg.Timeframes[len(cfg.Timeframes)-1] - } - - return &store.StrategyConfig{ - CoinSource: store.CoinSourceConfig{ - SourceType: "static", - StaticCoins: cfg.Symbols, - UseAI500: false, - AI500Limit: len(cfg.Symbols), - UseOITop: false, - OITopLimit: 0, - }, - Indicators: store.IndicatorConfig{ - Klines: store.KlineConfig{ - PrimaryTimeframe: primaryTF, - PrimaryCount: 30, - LongerTimeframe: longerTF, - LongerCount: 10, - EnableMultiTimeframe: len(cfg.Timeframes) > 1, - SelectedTimeframes: cfg.Timeframes, - }, - EnableRawKlines: true, - EnableEMA: true, - EnableMACD: true, - EnableRSI: true, - EnableATR: true, - EnableVolume: true, - EnableOI: true, - EnableFundingRate: true, - EMAPeriods: []int{20, 50}, - RSIPeriods: []int{7, 14}, - ATRPeriods: []int{14}, - }, - CustomPrompt: cfg.CustomPrompt, - RiskControl: store.RiskControlConfig{ - MaxPositions: 3, - BTCETHMaxLeverage: cfg.Leverage.BTCETHLeverage, - AltcoinMaxLeverage: cfg.Leverage.AltcoinLeverage, - BTCETHMaxPositionValueRatio: 5.0, - AltcoinMaxPositionValueRatio: 1.0, - MaxMarginUsage: 0.9, - MinPositionSize: 12, - MinRiskRewardRatio: 3.0, - MinConfidence: 75, - }, - } -} diff --git a/backtest/datafeed.go b/backtest/datafeed.go deleted file mode 100644 index 0e06ed82..00000000 --- a/backtest/datafeed.go +++ /dev/null @@ -1,206 +0,0 @@ -package backtest - -import ( - "fmt" - "sort" - "time" - - "nofx/market" -) - -type timeframeSeries struct { - klines []market.Kline - closeTimes []int64 -} - -type symbolSeries struct { - byTF map[string]*timeframeSeries -} - -// DataFeed manages historical kline data and provides time-progressive snapshots for backtesting. -type DataFeed struct { - cfg BacktestConfig - symbols []string - timeframes []string - symbolSeries map[string]*symbolSeries - decisionTimes []int64 - primaryTF string - longerTF string -} - -func NewDataFeed(cfg BacktestConfig) (*DataFeed, error) { - df := &DataFeed{ - cfg: cfg, - symbols: make([]string, len(cfg.Symbols)), - timeframes: append([]string(nil), cfg.Timeframes...), - symbolSeries: make(map[string]*symbolSeries), - primaryTF: cfg.DecisionTimeframe, - } - copy(df.symbols, cfg.Symbols) - - if err := df.loadAll(); err != nil { - return nil, err - } - - return df, nil -} - -func (df *DataFeed) loadAll() error { - start := time.Unix(df.cfg.StartTS, 0) - end := time.Unix(df.cfg.EndTS, 0) - - // longest timeframe used for auxiliary indicators - var longestDur time.Duration - for _, tf := range df.timeframes { - dur, err := market.TFDuration(tf) - if err != nil { - return err - } - if dur > longestDur { - longestDur = dur - df.longerTF = tf - } - } - - for _, symbol := range df.symbols { - ss := &symbolSeries{byTF: make(map[string]*timeframeSeries)} - for _, tf := range df.timeframes { - dur, _ := market.TFDuration(tf) - buffer := dur * 200 - fetchStart := start.Add(-buffer) - if fetchStart.Before(time.Unix(0, 0)) { - fetchStart = time.Unix(0, 0) - } - fetchEnd := end.Add(dur) - - klines, err := market.GetKlinesRange(symbol, tf, fetchStart, fetchEnd) - if err != nil { - return fmt.Errorf("fetch klines for %s %s: %w", symbol, tf, err) - } - if len(klines) == 0 { - return fmt.Errorf("no klines for %s %s", symbol, tf) - } - - series := &timeframeSeries{ - klines: klines, - closeTimes: make([]int64, len(klines)), - } - for i, k := range klines { - series.closeTimes[i] = k.CloseTime - } - ss.byTF[tf] = series - } - df.symbolSeries[symbol] = ss - } - - // Generate backtest progress timeline using the primary timeframe of the first symbol - firstSymbol := df.symbols[0] - primarySeries := df.symbolSeries[firstSymbol].byTF[df.primaryTF] - startMs := start.UnixMilli() - endMs := end.UnixMilli() - for _, ts := range primarySeries.closeTimes { - if ts < startMs { - continue - } - if ts > endMs { - break - } - df.decisionTimes = append(df.decisionTimes, ts) - // Align other symbols; report error early if data is missing - for _, symbol := range df.symbols[1:] { - if _, ok := df.symbolSeries[symbol].byTF[df.primaryTF]; !ok { - return fmt.Errorf("symbol %s missing timeframe %s", symbol, df.primaryTF) - } - } - } - if len(df.decisionTimes) == 0 { - return fmt.Errorf("no decision bars in range") - } - return nil -} - -func (df *DataFeed) DecisionBarCount() int { - return len(df.decisionTimes) -} - -func (df *DataFeed) DecisionTimestamp(index int) int64 { - // Bounds check to prevent panic - if index < 0 || index >= len(df.decisionTimes) { - return 0 - } - return df.decisionTimes[index] -} - -func (df *DataFeed) sliceUpTo(symbol, tf string, ts int64) []market.Kline { - // Nil checks to prevent panic - ss, ok := df.symbolSeries[symbol] - if !ok || ss == nil { - return nil - } - series, ok := ss.byTF[tf] - if !ok || series == nil { - return nil - } - idx := sort.Search(len(series.closeTimes), func(i int) bool { - return series.closeTimes[i] > ts - }) - if idx <= 0 { - return nil - } - return series.klines[:idx] -} - -func (df *DataFeed) BuildMarketData(ts int64) (map[string]*market.Data, map[string]map[string]*market.Data, error) { - result := make(map[string]*market.Data, len(df.symbols)) - multi := make(map[string]map[string]*market.Data, len(df.symbols)) - - for _, symbol := range df.symbols { - perTF := make(map[string]*market.Data, len(df.timeframes)) - for _, tf := range df.timeframes { - series := df.sliceUpTo(symbol, tf, ts) - if len(series) == 0 { - continue - } - var longer []market.Kline - if df.longerTF != "" && df.longerTF != tf { - longer = df.sliceUpTo(symbol, df.longerTF, ts) - } - data, err := market.BuildDataFromKlines(symbol, series, longer) - if err != nil { - return nil, nil, err - } - perTF[tf] = data - if tf == df.primaryTF { - result[symbol] = data - } - } - if _, ok := perTF[df.primaryTF]; !ok { - return nil, nil, fmt.Errorf("no primary data for %s at %d", symbol, ts) - } - multi[symbol] = perTF - } - return result, multi, nil -} - -func (df *DataFeed) decisionBarSnapshot(symbol string, ts int64) (*market.Kline, *market.Kline) { - ss, ok := df.symbolSeries[symbol] - if !ok { - return nil, nil - } - series, ok := ss.byTF[df.primaryTF] - if !ok { - return nil, nil - } - idx := sort.Search(len(series.closeTimes), func(i int) bool { - return series.closeTimes[i] >= ts - }) - if idx >= len(series.closeTimes) || series.closeTimes[idx] != ts { - return nil, nil - } - curr := &series.klines[idx] - var next *market.Kline - if idx+1 < len(series.klines) { - next = &series.klines[idx+1] - } - return curr, next -} diff --git a/backtest/equity.go b/backtest/equity.go deleted file mode 100644 index 8f153268..00000000 --- a/backtest/equity.go +++ /dev/null @@ -1,95 +0,0 @@ -package backtest - -import ( - "math" - "sort" - - "nofx/market" -) - -// ResampleEquity resamples equity curve based on timeframe. -func ResampleEquity(points []EquityPoint, timeframe string) ([]EquityPoint, error) { - if timeframe == "" { - return points, nil - } - dur, err := market.TFDuration(timeframe) - if err != nil { - return nil, err - } - if len(points) == 0 { - return points, nil - } - - durMs := dur.Milliseconds() - if durMs <= 0 { - return points, nil - } - - bucketMap := make(map[int64]EquityPoint) - bucketKeys := make([]int64, 0) - for _, pt := range points { - bucket := (pt.Timestamp / durMs) * durMs - if _, exists := bucketMap[bucket]; !exists { - bucketKeys = append(bucketKeys, bucket) - } - bucketPoint := pt - bucketPoint.Timestamp = bucket - bucketMap[bucket] = bucketPoint - } - - sort.Slice(bucketKeys, func(i, j int) bool { - return bucketKeys[i] < bucketKeys[j] - }) - - resampled := make([]EquityPoint, 0, len(bucketKeys)) - for _, key := range bucketKeys { - resampled = append(resampled, bucketMap[key]) - } - - return resampled, nil -} - -// LimitEquityPoints limits the number of data points within a given range (uniform sampling). -func LimitEquityPoints(points []EquityPoint, limit int) []EquityPoint { - if limit <= 0 || len(points) <= limit { - return points - } - - step := float64(len(points)) / float64(limit) - result := make([]EquityPoint, 0, limit) - for i := 0; i < limit; i++ { - idx := int(math.Round(step * float64(i))) - if idx >= len(points) { - idx = len(points) - 1 - } - result = append(result, points[idx]) - } - - return result -} - -// LimitTradeEvents applies uniform sampling to trade events. -func LimitTradeEvents(events []TradeEvent, limit int) []TradeEvent { - if limit <= 0 || len(events) <= limit { - return events - } - - step := float64(len(events)) / float64(limit) - result := make([]TradeEvent, 0, limit) - for i := 0; i < limit; i++ { - idx := int(math.Round(step * float64(i))) - if idx >= len(events) { - idx = len(events) - 1 - } - result = append(result, events[idx]) - } - return result -} - -// AlignEquityTimestamps ensures timestamps are sorted in ascending order. -func AlignEquityTimestamps(points []EquityPoint) []EquityPoint { - sort.Slice(points, func(i, j int) bool { - return points[i].Timestamp < points[j].Timestamp - }) - return points -} diff --git a/backtest/lock.go b/backtest/lock.go deleted file mode 100644 index 6f26ed08..00000000 --- a/backtest/lock.go +++ /dev/null @@ -1,100 +0,0 @@ -package backtest - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "time" -) - -const ( - lockFileName = "lock" - lockHeartbeatInterval = 2 * time.Second - lockStaleAfter = 10 * time.Second -) - -// RunLockInfo represents the lock file structure for a backtest run. -type RunLockInfo struct { - RunID string `json:"run_id"` - PID int `json:"pid"` - Host string `json:"host"` - StartedAt time.Time `json:"started_at"` - LastHeartbeat time.Time `json:"last_heartbeat"` -} - -func lockFilePath(runID string) string { - return filepath.Join(runDir(runID), lockFileName) -} - -func loadRunLock(runID string) (*RunLockInfo, error) { - path := lockFilePath(runID) - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var info RunLockInfo - if err := json.Unmarshal(data, &info); err != nil { - return nil, err - } - return &info, nil -} - -func saveRunLock(info *RunLockInfo) error { - if info == nil { - return fmt.Errorf("lock info nil") - } - return writeJSONAtomic(lockFilePath(info.RunID), info) -} - -func deleteRunLock(runID string) error { - err := os.Remove(lockFilePath(runID)) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil -} - -func lockIsStale(info *RunLockInfo) bool { - if info == nil { - return true - } - return time.Since(info.LastHeartbeat) > lockStaleAfter -} - -func acquireRunLock(runID string) (*RunLockInfo, error) { - if err := ensureRunDir(runID); err != nil { - return nil, err - } - - if existing, err := loadRunLock(runID); err == nil { - if !lockIsStale(existing) { - return nil, fmt.Errorf("run %s is locked by pid %d", runID, existing.PID) - } - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } - - host, _ := os.Hostname() - info := &RunLockInfo{ - RunID: runID, - PID: os.Getpid(), - Host: host, - StartedAt: time.Now().UTC(), - LastHeartbeat: time.Now().UTC(), - } - - if err := saveRunLock(info); err != nil { - return nil, err - } - return info, nil -} - -func updateRunLockHeartbeat(info *RunLockInfo) error { - if info == nil { - return fmt.Errorf("lock info nil") - } - info.LastHeartbeat = time.Now().UTC() - return saveRunLock(info) -} diff --git a/backtest/manager.go b/backtest/manager.go deleted file mode 100644 index 34ec9ec2..00000000 --- a/backtest/manager.go +++ /dev/null @@ -1,493 +0,0 @@ -package backtest - -import ( - "context" - "errors" - "fmt" - "nofx/logger" - "os" - "sort" - "strings" - "sync" - - "nofx/mcp" - "nofx/store" -) - -type Manager struct { - mu sync.RWMutex - runners map[string]*Runner - metadata map[string]*RunMetadata - cancels map[string]context.CancelFunc - mcpClient mcp.AIClient - aiResolver AIConfigResolver -} - -type AIConfigResolver func(*BacktestConfig) error - -func NewManager(defaultClient mcp.AIClient) *Manager { - return &Manager{ - runners: make(map[string]*Runner), - metadata: make(map[string]*RunMetadata), - cancels: make(map[string]context.CancelFunc), - mcpClient: defaultClient, - } -} - -func (m *Manager) SetAIResolver(resolver AIConfigResolver) { - m.mu.Lock() - defer m.mu.Unlock() - m.aiResolver = resolver -} - -func (m *Manager) Start(ctx context.Context, cfg BacktestConfig) (*Runner, error) { - if err := cfg.Validate(); err != nil { - return nil, err - } - if err := m.resolveAIConfig(&cfg); err != nil { - return nil, err - } - if ctx == nil { - ctx = context.Background() - } - - m.mu.Lock() - if existing, ok := m.runners[cfg.RunID]; ok { - state := existing.Status() - if state == RunStateRunning || state == RunStatePaused { - m.mu.Unlock() - return nil, fmt.Errorf("run %s is already active", cfg.RunID) - } - } - m.mu.Unlock() - - persistCfg := cfg - persistCfg.AICfg.APIKey = "" - if err := SaveConfig(cfg.RunID, &persistCfg); err != nil { - return nil, err - } - - runner, err := NewRunner(cfg, m.client()) - if err != nil { - return nil, err - } - - runCtx, cancel := context.WithCancel(ctx) - - m.mu.Lock() - if _, exists := m.runners[cfg.RunID]; exists { - m.mu.Unlock() - cancel() - return nil, fmt.Errorf("run %s is already active", cfg.RunID) - } - m.runners[cfg.RunID] = runner - m.cancels[cfg.RunID] = cancel - meta := runner.CurrentMetadata() - m.metadata[cfg.RunID] = meta - m.mu.Unlock() - - if err := runner.Start(runCtx); err != nil { - cancel() - m.mu.Lock() - delete(m.runners, cfg.RunID) - delete(m.cancels, cfg.RunID) - delete(m.metadata, cfg.RunID) - m.mu.Unlock() - runner.releaseLock() - return nil, err - } - - m.storeMetadata(cfg.RunID, meta) - m.launchWatcher(cfg.RunID, runner) - return runner, nil -} - -func (m *Manager) client() mcp.AIClient { - if m.mcpClient != nil { - return m.mcpClient - } - return mcp.New() -} - -func (m *Manager) GetRunner(runID string) (*Runner, bool) { - m.mu.RLock() - runner, ok := m.runners[runID] - m.mu.RUnlock() - return runner, ok -} - -func (m *Manager) ListRuns() ([]*RunMetadata, error) { - m.mu.RLock() - localCopy := make(map[string]*RunMetadata, len(m.metadata)) - for k, v := range m.metadata { - localCopy[k] = v - } - m.mu.RUnlock() - - runIDs, err := LoadRunIDs() - if err != nil { - return nil, err - } - - ordered := make([]string, 0, len(runIDs)) - if entries, err := listIndexEntries(); err == nil { - seen := make(map[string]bool, len(runIDs)) - for _, entry := range entries { - if contains(runIDs, entry.RunID) { - ordered = append(ordered, entry.RunID) - seen[entry.RunID] = true - } - } - for _, id := range runIDs { - if !seen[id] { - ordered = append(ordered, id) - } - } - } else { - ordered = append(ordered, runIDs...) - } - - metas := make([]*RunMetadata, 0, len(runIDs)) - for _, runID := range ordered { - if meta, ok := localCopy[runID]; ok { - metas = append(metas, meta) - continue - } - meta, err := LoadRunMetadata(runID) - if err == nil { - metas = append(metas, meta) - } - } - - sort.Slice(metas, func(i, j int) bool { - return metas[i].UpdatedAt.After(metas[j].UpdatedAt) - }) - - return metas, nil -} - -func contains(list []string, target string) bool { - for _, item := range list { - if item == target { - return true - } - } - return false -} - -func (m *Manager) Pause(runID string) error { - runner, ok := m.GetRunner(runID) - if !ok { - return fmt.Errorf("run %s not found", runID) - } - runner.Pause() - m.refreshMetadata(runID) - return nil -} - -func (m *Manager) Resume(runID string) error { - if runID == "" { - return fmt.Errorf("run_id is required") - } - - runner, ok := m.GetRunner(runID) - if ok { - runner.Resume() - m.refreshMetadata(runID) - return nil - } - - cfg, err := LoadConfig(runID) - if err != nil { - return err - } - cfgCopy := *cfg - if err := cfgCopy.Validate(); err != nil { - return err - } - if err := m.resolveAIConfig(&cfgCopy); err != nil { - return err - } - - restored, err := NewRunner(cfgCopy, m.client()) - if err != nil { - return err - } - if err := restored.RestoreFromCheckpoint(); err != nil { - return err - } - - ctx, cancel := context.WithCancel(context.Background()) - - m.mu.Lock() - if _, exists := m.runners[runID]; exists { - m.mu.Unlock() - cancel() - return fmt.Errorf("run %s is already active", runID) - } - m.runners[runID] = restored - m.cancels[runID] = cancel - m.metadata[runID] = restored.CurrentMetadata() - m.mu.Unlock() - - if err := restored.Start(ctx); err != nil { - cancel() - m.mu.Lock() - delete(m.runners, runID) - delete(m.cancels, runID) - delete(m.metadata, runID) - m.mu.Unlock() - restored.releaseLock() - return err - } - - m.storeMetadata(runID, restored.CurrentMetadata()) - m.launchWatcher(runID, restored) - return nil -} - -func (m *Manager) Stop(runID string) error { - runner, ok := m.GetRunner(runID) - if ok { - runner.Stop() - err := runner.Wait() - m.refreshMetadata(runID) - return err - } - meta, err := m.LoadMetadata(runID) - if err != nil { - return err - } - if meta.State == RunStateStopped || meta.State == RunStateCompleted { - return nil - } - meta.State = RunStateStopped - m.storeMetadata(runID, meta) - return nil -} - -func (m *Manager) Wait(runID string) error { - runner, ok := m.GetRunner(runID) - if !ok { - return fmt.Errorf("run %s not found", runID) - } - err := runner.Wait() - m.refreshMetadata(runID) - return err -} - -func (m *Manager) UpdateLabel(runID, label string) (*RunMetadata, error) { - meta, err := m.LoadMetadata(runID) - if err != nil { - return nil, err - } - clean := strings.TrimSpace(label) - metaCopy := *meta - metaCopy.Label = clean - m.storeMetadata(runID, &metaCopy) - return &metaCopy, nil -} - -func (m *Manager) Delete(runID string) error { - runner, ok := m.GetRunner(runID) - if ok { - runner.Stop() - _ = runner.Wait() - } - m.mu.Lock() - if cancel, ok := m.cancels[runID]; ok { - cancel() - delete(m.cancels, runID) - } - delete(m.runners, runID) - delete(m.metadata, runID) - m.mu.Unlock() - if err := removeFromRunIndex(runID); err != nil { - return err - } - if err := deleteRunLock(runID); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - return nil -} - -func (m *Manager) LoadMetadata(runID string) (*RunMetadata, error) { - runner, ok := m.GetRunner(runID) - if ok { - meta := runner.CurrentMetadata() - m.storeMetadata(runID, meta) - return meta, nil - } - meta, err := LoadRunMetadata(runID) - if err != nil { - return nil, err - } - m.storeMetadata(runID, meta) - return meta, nil -} - -func (m *Manager) LoadEquity(runID string, timeframe string, limit int) ([]EquityPoint, error) { - points, err := LoadEquityPoints(runID) - if err != nil { - return nil, err - } - if timeframe != "" { - points, err = ResampleEquity(points, timeframe) - if err != nil { - return nil, err - } - } - points = AlignEquityTimestamps(points) - points = LimitEquityPoints(points, limit) - return points, nil -} - -func (m *Manager) LoadTrades(runID string, limit int) ([]TradeEvent, error) { - events, err := LoadTradeEvents(runID) - if err != nil { - return nil, err - } - return LimitTradeEvents(events, limit), nil -} - -func (m *Manager) GetMetrics(runID string) (*Metrics, error) { - return LoadMetrics(runID) -} - -func (m *Manager) Cleanup(runID string) { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.runners, runID) - if cancel, ok := m.cancels[runID]; ok { - cancel() - delete(m.cancels, runID) - } -} - -func (m *Manager) Status(runID string) *StatusPayload { - runner, ok := m.GetRunner(runID) - if !ok { - return nil - } - payload := runner.StatusPayload() - m.storeMetadata(runID, runner.CurrentMetadata()) - return &payload -} - -func (m *Manager) launchWatcher(runID string, runner *Runner) { - go func() { - if err := runner.Wait(); err != nil { - logger.Infof("backtest run %s finished with error: %v", runID, err) - } - runner.PersistMetadata() - meta := runner.CurrentMetadata() - m.storeMetadata(runID, meta) - - m.mu.Lock() - if cancel, ok := m.cancels[runID]; ok { - cancel() - delete(m.cancels, runID) - } - delete(m.runners, runID) - m.mu.Unlock() - }() -} - -func (m *Manager) refreshMetadata(runID string) { - runner, ok := m.GetRunner(runID) - if !ok { - return - } - meta := runner.CurrentMetadata() - m.storeMetadata(runID, meta) -} - -func (m *Manager) storeMetadata(runID string, meta *RunMetadata) { - if meta == nil { - return - } - m.mu.Lock() - if existing, ok := m.metadata[runID]; ok { - if meta.Label == "" && existing.Label != "" { - meta.Label = existing.Label - } - if meta.LastError == "" && existing.LastError != "" { - meta.LastError = existing.LastError - } - } - m.metadata[runID] = meta - m.mu.Unlock() - _ = SaveRunMetadata(meta) - if err := updateRunIndex(meta, nil); err != nil { - logger.Infof("failed to update run index for %s: %v", runID, err) - } -} - -func (m *Manager) resolveAIConfig(cfg *BacktestConfig) error { - if cfg == nil { - return fmt.Errorf("ai config missing") - } - provider := strings.TrimSpace(cfg.AICfg.Provider) - apiKey := strings.TrimSpace(cfg.AICfg.APIKey) - if provider != "" && !strings.EqualFold(provider, "inherit") && apiKey != "" { - return nil - } - - m.mu.RLock() - resolver := m.aiResolver - m.mu.RUnlock() - if resolver == nil { - if apiKey == "" { - return fmt.Errorf("AI configuration missing key and no resolver configured") - } - return nil - } - return resolver(cfg) -} - -func (m *Manager) GetTrace(runID string, cycle int) (*store.DecisionRecord, error) { - return LoadDecisionTrace(runID, cycle) -} - -func (m *Manager) ExportRun(runID string) (string, error) { - return CreateRunExport(runID) -} - -// RestoreRuns scans the backtests directory and restores metadata for existing runs (service restart scenario). -func (m *Manager) RestoreRuns() error { - runIDs, err := LoadRunIDs() - if err != nil { - return err - } - for _, runID := range runIDs { - meta, err := LoadRunMetadata(runID) - if err != nil { - logger.Infof("skip run %s: %v", runID, err) - continue - } - if meta.State == RunStateRunning { - lock, err := loadRunLock(runID) - if err != nil || lockIsStale(lock) { - if err := deleteRunLock(runID); err != nil { - logger.Infof("failed to cleanup lock for %s: %v", runID, err) - } - meta.State = RunStatePaused - if err := SaveRunMetadata(meta); err != nil { - logger.Infof("failed to mark %s paused: %v", runID, err) - } - } - } - m.mu.Lock() - m.metadata[runID] = meta - m.mu.Unlock() - if err := updateRunIndex(meta, nil); err != nil { - logger.Infof("failed to sync index for %s: %v", runID, err) - } - } - return nil -} - -// RestoreRunsFromDisk retains the old method name for backward compatibility. -func (m *Manager) RestoreRunsFromDisk() error { - return m.RestoreRuns() -} diff --git a/backtest/metrics.go b/backtest/metrics.go deleted file mode 100644 index a7aac519..00000000 --- a/backtest/metrics.go +++ /dev/null @@ -1,239 +0,0 @@ -package backtest - -import ( - "fmt" - "math" - "strings" -) - -// CalculateMetrics reads existing logs and calculates summary metrics. state is optional, used to supplement information not yet persisted. -func CalculateMetrics(runID string, cfg *BacktestConfig, state *BacktestState) (*Metrics, error) { - if cfg == nil { - return nil, fmt.Errorf("config is nil") - } - - points, err := LoadEquityPoints(runID) - if err != nil { - return nil, fmt.Errorf("load equity points: %w", err) - } - - events, err := LoadTradeEvents(runID) - if err != nil { - return nil, fmt.Errorf("load trade events: %w", err) - } - - metrics := &Metrics{ - SymbolStats: make(map[string]SymbolMetrics), - } - - metrics.Liquidated = determineLiquidation(events, state) - - initialBalance := cfg.InitialBalance - if initialBalance <= 0 { - initialBalance = 1 - } - - lastEquity := initialBalance - if len(points) > 0 && points[len(points)-1].Equity > 0 { - lastEquity = points[len(points)-1].Equity - } else if state != nil && state.Equity > 0 { - lastEquity = state.Equity - } - metrics.TotalReturnPct = ((lastEquity - initialBalance) / initialBalance) * 100 - - metrics.MaxDrawdownPct = maxDrawdown(points, state) - metrics.SharpeRatio = sharpeRatio(points) - - fillTradeMetrics(metrics, events) - - return metrics, nil -} - -func determineLiquidation(events []TradeEvent, state *BacktestState) bool { - if state != nil && state.Liquidated { - return true - } - for i := len(events) - 1; i >= 0; i-- { - if events[i].LiquidationFlag { - return true - } - } - return false -} - -func maxDrawdown(points []EquityPoint, state *BacktestState) float64 { - if len(points) == 0 { - if state != nil { - return state.MaxDrawdownPct - } - return 0 - } - peak := points[0].Equity - if peak <= 0 { - peak = 1 - } - maxDD := 0.0 - for _, pt := range points { - if pt.Equity > peak { - peak = pt.Equity - } - if peak <= 0 { - continue - } - dd := (peak - pt.Equity) / peak * 100 - if dd > maxDD { - maxDD = dd - } - } - if state != nil && state.MaxDrawdownPct > maxDD { - maxDD = state.MaxDrawdownPct - } - return maxDD -} - -// sharpeRatio calculates the Sharpe ratio from equity points. -// Uses sample standard deviation (n-1) and annualizes assuming ~252 trading days. -// Returns math.NaN() for edge cases (insufficient data, zero variance). -func sharpeRatio(points []EquityPoint) float64 { - // Need at least 10 data points for meaningful Sharpe calculation - const minDataPoints = 10 - if len(points) < minDataPoints { - return 0 - } - - returns := make([]float64, 0, len(points)-1) - prev := points[0].Equity - for i := 1; i < len(points); i++ { - curr := points[i].Equity - if prev <= 0 { - prev = curr - continue - } - ret := (curr - prev) / prev - returns = append(returns, ret) - prev = curr - } - if len(returns) < minDataPoints-1 { - return 0 - } - - // Calculate mean return - mean := 0.0 - for _, r := range returns { - mean += r - } - mean /= float64(len(returns)) - - // Calculate sample variance (using n-1 for unbiased estimator) - variance := 0.0 - for _, r := range returns { - diff := r - mean - variance += diff * diff - } - if len(returns) > 1 { - variance /= float64(len(returns) - 1) - } - - std := math.Sqrt(variance) - if std < 1e-10 { - // Zero or near-zero volatility - return 0 instead of infinity/NaN - return 0 - } - - // Calculate Sharpe ratio (assuming risk-free rate = 0 for crypto) - // Annualize by multiplying by sqrt(periods per year) - // Assuming each equity point represents ~1 hour, we have ~8760 periods/year - // For conservative estimate, use sqrt(252) as if daily returns - periodsPerYear := 252.0 - annualizationFactor := math.Sqrt(periodsPerYear) - - sharpe := (mean / std) * annualizationFactor - return sharpe -} - -func fillTradeMetrics(metrics *Metrics, events []TradeEvent) { - if metrics == nil { - return - } - - totalTrades := 0 - winTrades := 0 - lossTrades := 0 - totalWinAmount := 0.0 - totalLossAmount := 0.0 - - for _, evt := range events { - include := evt.LiquidationFlag || strings.HasPrefix(evt.Action, "close") - if evt.RealizedPnL != 0 { - include = true - } - if !include { - continue - } - totalTrades++ - - stats := metrics.SymbolStats[evt.Symbol] - stats.TotalTrades++ - stats.TotalPnL += evt.RealizedPnL - - if evt.RealizedPnL > 0 { - winTrades++ - totalWinAmount += evt.RealizedPnL - stats.WinningTrades++ - } else if evt.RealizedPnL < 0 { - lossTrades++ - totalLossAmount += -evt.RealizedPnL - stats.LosingTrades++ - } - - metrics.SymbolStats[evt.Symbol] = stats - } - - metrics.Trades = totalTrades - if totalTrades > 0 { - metrics.WinRate = (float64(winTrades) / float64(totalTrades)) * 100 - } - if winTrades > 0 { - metrics.AvgWin = totalWinAmount / float64(winTrades) - } - if lossTrades > 0 { - metrics.AvgLoss = -(totalLossAmount / float64(lossTrades)) - } - if totalLossAmount > 0 { - metrics.ProfitFactor = totalWinAmount / totalLossAmount - } else if totalWinAmount > 0 { - // No losses but have wins - use a high but reasonable cap - metrics.ProfitFactor = 100.0 - } - - bestSymbol := "" - bestPnL := math.Inf(-1) - worstSymbol := "" - worstPnL := math.Inf(1) - - for symbol, stats := range metrics.SymbolStats { - if stats.TotalTrades > 0 { - if stats.TotalPnL > bestPnL { - bestPnL = stats.TotalPnL - bestSymbol = symbol - } - if stats.TotalPnL < worstPnL { - worstPnL = stats.TotalPnL - worstSymbol = symbol - } - - stats.AvgPnL = stats.TotalPnL / float64(stats.TotalTrades) - stats.WinRate = (float64(stats.WinningTrades) / float64(stats.TotalTrades)) * 100 - } - metrics.SymbolStats[symbol] = stats - } - - metrics.BestSymbol = bestSymbol - if math.IsInf(bestPnL, -1) { - metrics.BestSymbol = "" - } - metrics.WorstSymbol = worstSymbol - if math.IsInf(worstPnL, 1) { - metrics.WorstSymbol = "" - } -} diff --git a/backtest/persistence_db.go b/backtest/persistence_db.go deleted file mode 100644 index d494ed65..00000000 --- a/backtest/persistence_db.go +++ /dev/null @@ -1,40 +0,0 @@ -package backtest - -import ( - "database/sql" - "fmt" - "strings" -) - -var persistenceDB *sql.DB -var dbIsPostgres bool - -// UseDatabase enables database-backed persistence for all backtest storage operations. -// If isPostgres is true, queries will use $1, $2... placeholders instead of ? -func UseDatabase(db *sql.DB) { - persistenceDB = db -} - -// UseDatabaseWithType enables database-backed persistence with explicit type. -func UseDatabaseWithType(db *sql.DB, isPostgres bool) { - persistenceDB = db - dbIsPostgres = isPostgres -} - -func usingDB() bool { - return persistenceDB != nil -} - -// convertQuery converts ? placeholders to $1, $2, etc. for PostgreSQL -func convertQuery(query string) string { - if !dbIsPostgres { - return query - } - result := query - index := 1 - for strings.Contains(result, "?") { - result = strings.Replace(result, "?", fmt.Sprintf("$%d", index), 1) - index++ - } - return result -} diff --git a/backtest/registry.go b/backtest/registry.go deleted file mode 100644 index 9c0330e6..00000000 --- a/backtest/registry.go +++ /dev/null @@ -1,160 +0,0 @@ -package backtest - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "time" -) - -const runIndexFile = "index.json" - -type RunIndexEntry struct { - RunID string `json:"run_id"` - State RunState `json:"state"` - Symbols []string `json:"symbols"` - DecisionTF string `json:"decision_tf"` - StartTS int64 `json:"start_ts"` - EndTS int64 `json:"end_ts"` - EquityLast float64 `json:"equity_last"` - MaxDrawdownPct float64 `json:"max_dd_pct"` - CreatedAtISO string `json:"created_at"` - UpdatedAtISO string `json:"updated_at"` -} - -type RunIndex struct { - Runs map[string]RunIndexEntry `json:"runs"` - UpdatedAt string `json:"updated_at"` -} - -func runIndexPath() string { - return filepath.Join(backtestsRootDir, runIndexFile) -} - -func loadRunIndex() (*RunIndex, error) { - if usingDB() { - entries, err := listIndexEntriesDB() - if err != nil { - return nil, err - } - idx := &RunIndex{ - Runs: make(map[string]RunIndexEntry), - UpdatedAt: time.Now().UTC().Format(time.RFC3339), - } - for _, entry := range entries { - idx.Runs[entry.RunID] = entry - } - return idx, nil - } - path := runIndexPath() - data, err := os.ReadFile(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return &RunIndex{Runs: make(map[string]RunIndexEntry)}, nil - } - return nil, err - } - var idx RunIndex - if err := json.Unmarshal(data, &idx); err != nil { - return nil, err - } - if idx.Runs == nil { - idx.Runs = make(map[string]RunIndexEntry) - } - return &idx, nil -} - -func saveRunIndex(idx *RunIndex) error { - if usingDB() { - return nil - } - if idx == nil { - return fmt.Errorf("index is nil") - } - idx.UpdatedAt = time.Now().UTC().Format(time.RFC3339) - return writeJSONAtomic(runIndexPath(), idx) -} - -func updateRunIndex(meta *RunMetadata, cfg *BacktestConfig) error { - if usingDB() { - enforceRetention(maxCompletedRuns) - return nil - } - if meta == nil { - return fmt.Errorf("meta nil") - } - if cfg == nil { - var err error - cfg, err = LoadConfig(meta.RunID) - if err != nil { - return err - } - } - - idx, err := loadRunIndex() - if err != nil { - return err - } - - entry := RunIndexEntry{ - RunID: meta.RunID, - State: meta.State, - Symbols: append([]string(nil), cfg.Symbols...), - DecisionTF: meta.Summary.DecisionTF, - StartTS: cfg.StartTS, - EndTS: cfg.EndTS, - EquityLast: meta.Summary.EquityLast, - MaxDrawdownPct: meta.Summary.MaxDrawdownPct, - CreatedAtISO: meta.CreatedAt.Format(time.RFC3339), - UpdatedAtISO: meta.UpdatedAt.Format(time.RFC3339), - } - - if idx.Runs == nil { - idx.Runs = make(map[string]RunIndexEntry) - } - idx.Runs[meta.RunID] = entry - if err := saveRunIndex(idx); err != nil { - return err - } - enforceRetention(maxCompletedRuns) - return nil -} - -func removeFromRunIndex(runID string) error { - if usingDB() { - if err := deleteRunDB(runID); err != nil { - return err - } - return os.RemoveAll(runDir(runID)) - } - idx, err := loadRunIndex() - if err != nil { - return err - } - if idx.Runs == nil { - return nil - } - delete(idx.Runs, runID) - return saveRunIndex(idx) -} - -func listIndexEntries() ([]RunIndexEntry, error) { - if usingDB() { - return listIndexEntriesDB() - } - idx, err := loadRunIndex() - if err != nil { - return nil, err - } - entries := make([]RunIndexEntry, 0, len(idx.Runs)) - for _, entry := range idx.Runs { - entries = append(entries, entry) - } - sort.Slice(entries, func(i, j int) bool { - return entries[i].UpdatedAtISO > entries[j].UpdatedAtISO - }) - return entries, nil -} diff --git a/backtest/retention.go b/backtest/retention.go deleted file mode 100644 index 49ec3542..00000000 --- a/backtest/retention.go +++ /dev/null @@ -1,101 +0,0 @@ -package backtest - -import ( - "nofx/logger" - "os" - "sort" - "time" -) - -const maxCompletedRuns = 100 - -func enforceRetention(maxRuns int) { - if maxRuns <= 0 { - return - } - if usingDB() { - enforceRetentionDB(maxRuns) - return - } - idx, err := loadRunIndex() - if err != nil { - return - } - - type wrapped struct { - entry RunIndexEntry - updated time.Time - } - finalStates := map[RunState]bool{ - RunStateCompleted: true, - RunStateStopped: true, - RunStateFailed: true, - RunStateLiquidated: true, - } - - candidates := make([]wrapped, 0) - for _, entry := range idx.Runs { - if !finalStates[entry.State] { - continue - } - ts, err := time.Parse(time.RFC3339, entry.UpdatedAtISO) - if err != nil { - ts = time.Now() - } - candidates = append(candidates, wrapped{entry: entry, updated: ts}) - } - if len(candidates) <= maxRuns { - return - } - - sort.Slice(candidates, func(i, j int) bool { - return candidates[i].updated.Before(candidates[j].updated) - }) - - toRemove := len(candidates) - maxRuns - for i := 0; i < toRemove; i++ { - runID := candidates[i].entry.RunID - if err := os.RemoveAll(runDir(runID)); err != nil { - logger.Infof("failed to prune run %s: %v", runID, err) - continue - } - delete(idx.Runs, runID) - } - if err := saveRunIndex(idx); err != nil { - logger.Infof("failed to save index after pruning: %v", err) - } -} - -func enforceRetentionDB(maxRuns int) { - finalStates := []RunState{ - RunStateCompleted, - RunStateStopped, - RunStateFailed, - RunStateLiquidated, - } - query := convertQuery(` - SELECT run_id FROM backtest_runs - WHERE state IN (?, ?, ?, ?) - ORDER BY updated_at DESC - OFFSET ? - `) - rows, err := persistenceDB.Query(query, - finalStates[0], finalStates[1], finalStates[2], finalStates[3], maxRuns) - if err != nil { - return - } - defer rows.Close() - for rows.Next() { - var runID string - if err := rows.Scan(&runID); err != nil { - continue - } - if err := deleteRunDB(runID); err != nil { - logger.Infof("failed to remove run %s: %v", runID, err) - continue - } - if err := os.RemoveAll(runDir(runID)); err != nil { - logger.Infof("failed to remove run dir %s: %v", runID, err) - } - } -} diff --git a/backtest/runner.go b/backtest/runner.go deleted file mode 100644 index 7b31c97f..00000000 --- a/backtest/runner.go +++ /dev/null @@ -1,341 +0,0 @@ -package backtest - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "sync" - "time" - - "nofx/kernel" - "nofx/logger" - "nofx/mcp" -) - -var ( - errBacktestCompleted = errors.New("backtest completed") - errLiquidated = errors.New("account liquidated") -) - -const ( - metricsWriteInterval = 5 * time.Second - aiDecisionMaxRetries = 3 -) - -// Runner encapsulates the lifecycle of a single backtest run. -type Runner struct { - cfg BacktestConfig - feed *DataFeed - account *BacktestAccount - strategyEngine *kernel.StrategyEngine - - decisionLogDir string - mcpClient mcp.AIClient - - statusMu sync.RWMutex - status RunState - - stateMu sync.RWMutex - state *BacktestState - - pauseCh chan struct{} - resumeCh chan struct{} - stopCh chan struct{} - doneCh chan struct{} - - err error - errMu sync.RWMutex - lastError string - lastCheckpoint time.Time - createdAt time.Time - lastMetricsWrite time.Time - - aiCache *AICache - cachePath string - - lockInfo *RunLockInfo - lockStop chan struct{} - lockStopOnce sync.Once // Ensures lockStop is closed only once -} - -// NewRunner constructs a backtest runner. -func NewRunner(cfg BacktestConfig, mcpClient mcp.AIClient) (*Runner, error) { - if err := ensureRunDir(cfg.RunID); err != nil { - return nil, err - } - - client, err := configureMCPClient(cfg, mcpClient) - if err != nil { - return nil, err - } - - feed, err := NewDataFeed(cfg) - if err != nil { - return nil, err - } - - if err := os.MkdirAll(decisionLogDir(cfg.RunID), 0o755); err != nil { - return nil, err - } - - dLogDir := decisionLogDir(cfg.RunID) - account := NewBacktestAccount(cfg.InitialBalance, cfg.FeeBps, cfg.SlippageBps) - - createdAt := time.Now().UTC() - state := &BacktestState{ - Positions: make(map[string]PositionSnapshot), - Cash: account.Cash(), - Equity: cfg.InitialBalance, - UnrealizedPnL: 0, - RealizedPnL: 0, - MaxEquity: cfg.InitialBalance, - MinEquity: cfg.InitialBalance, - MaxDrawdownPct: 0, - LastUpdate: createdAt, - } - - var ( - aiCache *AICache - cachePath string - ) - if cfg.CacheAI || cfg.ReplayOnly || cfg.SharedAICachePath != "" { - cachePath = cfg.SharedAICachePath - if cachePath == "" { - cachePath = filepath.Join(runDir(cfg.RunID), "ai_cache.json") - } - cache, err := LoadAICache(cachePath) - if err != nil { - return nil, fmt.Errorf("load ai cache: %w", err) - } - aiCache = cache - } - - // Create strategy engine from backtest config for unified prompt generation - strategyConfig := cfg.ToStrategyConfig() - strategyEngine := kernel.NewStrategyEngine(strategyConfig) - - r := &Runner{ - cfg: cfg, - feed: feed, - account: account, - strategyEngine: strategyEngine, - decisionLogDir: dLogDir, - mcpClient: client, - status: RunStateCreated, - state: state, - pauseCh: make(chan struct{}, 1), - resumeCh: make(chan struct{}, 1), - stopCh: make(chan struct{}, 1), - doneCh: make(chan struct{}), - createdAt: createdAt, - aiCache: aiCache, - cachePath: cachePath, - } - - if err := r.initLock(); err != nil { - return nil, err - } - - return r, nil -} - -func (r *Runner) initLock() error { - if r.cfg.RunID == "" { - return fmt.Errorf("run_id required for lock") - } - info, err := acquireRunLock(r.cfg.RunID) - if err != nil { - return err - } - r.lockInfo = info - r.lockStop = make(chan struct{}) - go r.lockHeartbeatLoop() - return nil -} - -func (r *Runner) lockHeartbeatLoop() { - ticker := time.NewTicker(lockHeartbeatInterval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := updateRunLockHeartbeat(r.lockInfo); err != nil { - logger.Infof("failed to update lock heartbeat for %s: %v", r.cfg.RunID, err) - } - case <-r.lockStop: - return - } - } -} - -func (r *Runner) releaseLock() { - // Use sync.Once to ensure channel is closed exactly once, preventing panic on double-close - r.lockStopOnce.Do(func() { - if r.lockStop != nil { - close(r.lockStop) - } - }) - if err := deleteRunLock(r.cfg.RunID); err != nil { - logger.Infof("failed to release lock for %s: %v", r.cfg.RunID, err) - } - r.lockInfo = nil -} - -// Start launches the backtest loop. -func (r *Runner) Start(ctx context.Context) error { - r.statusMu.Lock() - if r.status != RunStateCreated && r.status != RunStatePaused { - r.statusMu.Unlock() - return fmt.Errorf("cannot start runner in state %s", r.status) - } - r.status = RunStateRunning - r.statusMu.Unlock() - - go r.loop(ctx) - return nil -} - -// PersistMetadata writes the current snapshot to run.json. -func (r *Runner) PersistMetadata() { - r.persistMetadata() -} - -func (r *Runner) setLastError(err error) { - r.errMu.Lock() - defer r.errMu.Unlock() - if err == nil { - r.lastError = "" - return - } - r.lastError = err.Error() -} - -func (r *Runner) lastErrorString() string { - r.errMu.RLock() - defer r.errMu.RUnlock() - return r.lastError -} - -// CurrentMetadata returns the metadata corresponding to the current in-memory state. -func (r *Runner) CurrentMetadata() *RunMetadata { - state := r.snapshotState() - meta := r.buildMetadata(state, r.Status()) - meta.CreatedAt = r.createdAt - meta.UpdatedAt = state.LastUpdate - return meta -} - -func (r *Runner) Pause() { - select { - case r.pauseCh <- struct{}{}: - default: - } -} - -func (r *Runner) Resume() { - select { - case r.resumeCh <- struct{}{}: - default: - } -} - -func (r *Runner) Stop() { - select { - case r.stopCh <- struct{}{}: - default: - } -} - -func (r *Runner) Wait() error { - <-r.doneCh - r.statusMu.RLock() - defer r.statusMu.RUnlock() - return r.err -} - -// Status returns the current run state. -func (r *Runner) Status() RunState { - r.statusMu.RLock() - defer r.statusMu.RUnlock() - return r.status -} - -// StatusPayload builds the status response for the API. -func (r *Runner) StatusPayload() StatusPayload { - snapshot := r.snapshotState() - progress := progressPercent(snapshot, r.cfg) - - // Build position statuses with unrealized P&L - positions := make([]PositionStatus, 0, len(snapshot.Positions)) - for _, pos := range snapshot.Positions { - if pos.Quantity <= 0 { - continue - } - // Get mark price from feed if available - markPrice := pos.AvgPrice // fallback to entry price - if r.feed != nil && snapshot.BarTimestamp > 0 { - if md, _, err := r.feed.BuildMarketData(snapshot.BarTimestamp); err == nil { - if data, ok := md[pos.Symbol]; ok { - markPrice = data.CurrentPrice - } - } - } - - // Calculate unrealized P&L - var unrealizedPnL float64 - if pos.Side == "long" { - unrealizedPnL = (markPrice - pos.AvgPrice) * pos.Quantity - } else { - unrealizedPnL = (pos.AvgPrice - markPrice) * pos.Quantity - } - - // Calculate P&L percentage based on margin - pnlPct := 0.0 - if pos.MarginUsed > 0 { - pnlPct = (unrealizedPnL / pos.MarginUsed) * 100 - } - - positions = append(positions, PositionStatus{ - Symbol: pos.Symbol, - Side: pos.Side, - Quantity: pos.Quantity, - EntryPrice: pos.AvgPrice, - MarkPrice: markPrice, - Leverage: pos.Leverage, - UnrealizedPnL: unrealizedPnL, - UnrealizedPnLPct: pnlPct, - MarginUsed: pos.MarginUsed, - }) - } - - payload := StatusPayload{ - RunID: r.cfg.RunID, - State: r.Status(), - ProgressPct: progress, - ProcessedBars: snapshot.BarIndex, - CurrentTime: snapshot.BarTimestamp, - DecisionCycle: snapshot.DecisionCycle, - Equity: snapshot.Equity, - UnrealizedPnL: snapshot.UnrealizedPnL, - RealizedPnL: snapshot.RealizedPnL, - Positions: positions, - Note: snapshot.LiquidationNote, - LastError: r.lastErrorString(), - LastUpdatedIso: snapshot.LastUpdate.UTC().Format(time.RFC3339), - } - return payload -} - -func (r *Runner) snapshotState() BacktestState { - r.stateMu.RLock() - defer r.stateMu.RUnlock() - - copyState := *r.state - copyState.Positions = make(map[string]PositionSnapshot, len(r.state.Positions)) - for k, v := range r.state.Positions { - copyState.Positions[k] = v - } - return copyState -} diff --git a/backtest/runner_loop.go b/backtest/runner_loop.go deleted file mode 100644 index 81703bc1..00000000 --- a/backtest/runner_loop.go +++ /dev/null @@ -1,563 +0,0 @@ -package backtest - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "time" - - "nofx/kernel" - "nofx/logger" - "nofx/market" - "nofx/store" -) - -func (r *Runner) loop(ctx context.Context) { - defer close(r.doneCh) - - for { - select { - case <-ctx.Done(): - r.handleStop(fmt.Errorf("context canceled: %w", ctx.Err())) - return - case <-r.stopCh: - r.handleStop(nil) - return - case <-r.pauseCh: - r.handlePause() - <-r.resumeCh - r.resumeFromPause() - default: - } - - err := r.stepOnce() - if errors.Is(err, errBacktestCompleted) { - r.handleCompletion() - return - } - if errors.Is(err, errLiquidated) { - r.handleLiquidation() - return - } - if err != nil { - r.handleFailure(err) - return - } - } -} - -func (r *Runner) stepOnce() error { - state := r.snapshotState() - if state.BarIndex >= r.feed.DecisionBarCount() { - return errBacktestCompleted - } - - ts := r.feed.DecisionTimestamp(state.BarIndex) - - marketData, multiTF, err := r.feed.BuildMarketData(ts) - if err != nil { - return err - } - - priceMap := make(map[string]float64, len(marketData)) - for symbol, data := range marketData { - priceMap[symbol] = data.CurrentPrice - } - - callCount := state.DecisionCycle + 1 - shouldDecide := r.shouldTriggerDecision(state.BarIndex) - - var ( - record *store.DecisionRecord - decisionActions []store.DecisionAction - tradeEvents = make([]TradeEvent, 0) - execLog []string - hadError bool - ) - - decisionAttempted := shouldDecide - - if shouldDecide { - ctx, rec, err := r.buildDecisionContext(ts, marketData, multiTF, priceMap, callCount) - if err != nil { - // Defensive nil check to prevent panic if buildDecisionContext returns error with nil record - if rec != nil { - rec.Success = false - rec.ErrorMessage = fmt.Sprintf("failed to build trading context: %v", err) - _ = r.logDecision(rec) - } - return err - } - record = rec - - var ( - fullDecision *kernel.FullDecision - fromCache bool - cacheKey string - ) - if r.aiCache != nil { - if key, err := computeCacheKey(ctx, r.cfg.PromptVariant, ts); err == nil { - cacheKey = key - if cached, ok := r.aiCache.Get(cacheKey); ok { - fullDecision = cached - fromCache = true - } else if r.cfg.ReplayOnly { - decisionErr := fmt.Errorf("replay_only enabled but cache miss at %d", ts) - record.Success = false - record.ErrorMessage = fmt.Sprintf("cached decision not found for ts=%d", ts) - _ = r.logDecision(record) - return decisionErr - } - } else { - logger.Infof("failed to compute ai cache key: %v", err) - } - } - - if !fromCache { - fd, err := r.invokeAIWithRetry(ctx) - if err != nil { - decisionAttempted = true - hadError = true - record.Success = false - record.ErrorMessage = fmt.Sprintf("AI decision failed: %v", err) - execLog = append(execLog, fmt.Sprintf("⚠️ AI decision failed: %v", err)) - r.setLastError(err) - } else { - fullDecision = fd - if r.cfg.CacheAI && r.aiCache != nil && cacheKey != "" { - if err := r.aiCache.Put(cacheKey, r.cfg.PromptVariant, ts, fullDecision); err != nil { - logger.Infof("failed to persist ai cache for %s: %v", r.cfg.RunID, err) - } - } - } - } - - if fullDecision != nil { - r.fillDecisionRecord(record, fullDecision) - - sorted := sortDecisionsByPriority(fullDecision.Decisions) - - prevLogs := execLog - decisionActions = make([]store.DecisionAction, 0, len(sorted)) - execLog = make([]string, 0, len(sorted)+len(prevLogs)) - if len(prevLogs) > 0 { - execLog = append(execLog, prevLogs...) - } - - for _, dec := range sorted { - actionRecord, trades, logEntry, execErr := r.executeDecision(dec, priceMap, ts, callCount) - if execErr != nil { - actionRecord.Success = false - actionRecord.Error = execErr.Error() - hadError = true - execLog = append(execLog, fmt.Sprintf("❌ %s %s: %v", dec.Symbol, dec.Action, execErr)) - } else { - actionRecord.Success = true - execLog = append(execLog, fmt.Sprintf("✓ %s %s", dec.Symbol, dec.Action)) - } - if len(trades) > 0 { - tradeEvents = append(tradeEvents, trades...) - } - if logEntry != "" { - execLog = append(execLog, logEntry) - } - decisionActions = append(decisionActions, actionRecord) - } - } - } - - cycleForLog := state.DecisionCycle - if decisionAttempted { - cycleForLog = callCount - } - - liquidationEvents, liquidationNote, err := r.checkLiquidation(ts, priceMap, cycleForLog) - if err != nil { - if record != nil { - record.Success = false - record.ErrorMessage = err.Error() - _ = r.logDecision(record) - } - return err - } - if len(liquidationEvents) > 0 { - hadError = true - tradeEvents = append(tradeEvents, liquidationEvents...) - if record != nil { - execLog = append(execLog, fmt.Sprintf("⚠️ Forced liquidation: %s", liquidationNote)) - } - } - - if record != nil { - record.Decisions = decisionActions - record.ExecutionLog = execLog - record.Success = !hadError && liquidationNote == "" - if liquidationNote != "" { - record.ErrorMessage = liquidationNote - } - } - - equity, unrealized, _ := r.account.TotalEquity(priceMap) - marginUsed := r.totalMarginUsed() - - r.updateState(ts, equity, unrealized, marginUsed, priceMap, decisionAttempted) - - snapshot := r.snapshotState() - drawdownPct := 0.0 - if snapshot.MaxEquity > 0 { - drawdownPct = ((snapshot.MaxEquity - snapshot.Equity) / snapshot.MaxEquity) * 100 - } - - equityPoint := EquityPoint{ - Timestamp: ts, - Equity: snapshot.Equity, - Available: snapshot.Cash, - PnL: snapshot.Equity - r.account.InitialBalance(), - PnLPct: ((snapshot.Equity - r.account.InitialBalance()) / r.account.InitialBalance()) * 100, - DrawdownPct: drawdownPct, - Cycle: snapshot.DecisionCycle, - } - - if err := appendEquityPoint(r.cfg.RunID, equityPoint); err != nil { - return err - } - - for _, evt := range tradeEvents { - if err := appendTradeEvent(r.cfg.RunID, evt); err != nil { - return err - } - } - - if record != nil { - if err := r.logDecision(record); err != nil { - return err - } - } - - if err := saveProgress(r.cfg.RunID, &snapshot, &r.cfg); err != nil { - return err - } - - if err := r.maybeCheckpoint(); err != nil { - return err - } - - r.persistMetadata() - r.persistMetrics(false) - - if !hadError && liquidationNote == "" { - r.setLastError(nil) - } - - if snapshot.Liquidated { - return errLiquidated - } - - return nil -} - -func (r *Runner) buildDecisionContext(ts int64, marketData map[string]*market.Data, multiTF map[string]map[string]*market.Data, priceMap map[string]float64, callCount int) (*kernel.Context, *store.DecisionRecord, error) { - equity, unrealized, _ := r.account.TotalEquity(priceMap) - available := r.account.Cash() - marginUsed := r.totalMarginUsed() - marginPct := 0.0 - if equity > 0 { - marginPct = (marginUsed / equity) * 100 - } - - accountInfo := kernel.AccountInfo{ - TotalEquity: equity, - AvailableBalance: available, - TotalPnL: equity - r.account.InitialBalance(), - TotalPnLPct: ((equity - r.account.InitialBalance()) / r.account.InitialBalance()) * 100, - MarginUsed: marginUsed, - MarginUsedPct: marginPct, - PositionCount: len(r.account.Positions()), - } - - positions := r.convertPositions(priceMap) - - // Get candidate coins from strategy engine (includes source info) - candidateCoins, err := r.strategyEngine.GetCandidateCoins() - if err != nil { - // Fallback to simple list if strategy engine fails - candidateCoins = make([]kernel.CandidateCoin, 0, len(r.cfg.Symbols)) - for _, sym := range r.cfg.Symbols { - candidateCoins = append(candidateCoins, kernel.CandidateCoin{Symbol: sym, Sources: []string{"backtest"}}) - } - } - - runtime := int((ts - int64(r.cfg.StartTS*1000)) / 60000) - ctx := &kernel.Context{ - CurrentTime: time.UnixMilli(ts).UTC().Format("2006-01-02 15:04:05 UTC"), - RuntimeMinutes: runtime, - CallCount: callCount, - Account: accountInfo, - Positions: positions, - CandidateCoins: candidateCoins, - PromptVariant: r.cfg.PromptVariant, - MarketDataMap: marketData, - MultiTFMarket: multiTF, - BTCETHLeverage: r.cfg.Leverage.BTCETHLeverage, - AltcoinLeverage: r.cfg.Leverage.AltcoinLeverage, - Timeframes: r.cfg.Timeframes, - } - - // Fetch quantitative data if enabled in strategy (uses current data as approximation) - strategyConfig := r.strategyEngine.GetConfig() - if strategyConfig.Indicators.EnableQuantData { - // Collect symbols to query (candidate coins + position coins) - symbolSet := make(map[string]bool) - for _, sym := range r.cfg.Symbols { - symbolSet[sym] = true - } - for _, pos := range positions { - symbolSet[pos.Symbol] = true - } - symbols := make([]string, 0, len(symbolSet)) - for sym := range symbolSet { - symbols = append(symbols, sym) - } - ctx.QuantDataMap = r.strategyEngine.FetchQuantDataBatch(symbols) - if len(ctx.QuantDataMap) > 0 { - logger.Infof("📊 Backtest: fetched quant data for %d symbols", len(ctx.QuantDataMap)) - } - } - - // Fetch OI ranking data if enabled in strategy (uses current data as approximation) - if strategyConfig.Indicators.EnableOIRanking { - ctx.OIRankingData = r.strategyEngine.FetchOIRankingData() - if ctx.OIRankingData != nil { - logger.Infof("📊 Backtest: OI ranking data ready: %d top, %d low positions", - len(ctx.OIRankingData.TopPositions), len(ctx.OIRankingData.LowPositions)) - } - } - - // Fetch NetFlow ranking data if enabled in strategy - if strategyConfig.Indicators.EnableNetFlowRanking { - ctx.NetFlowRankingData = r.strategyEngine.FetchNetFlowRankingData() - if ctx.NetFlowRankingData != nil { - logger.Infof("💰 Backtest: NetFlow ranking data ready: inst_in=%d, inst_out=%d", - len(ctx.NetFlowRankingData.InstitutionFutureTop), len(ctx.NetFlowRankingData.InstitutionFutureLow)) - } - } - - // Fetch Price ranking data if enabled in strategy - if strategyConfig.Indicators.EnablePriceRanking { - ctx.PriceRankingData = r.strategyEngine.FetchPriceRankingData() - if ctx.PriceRankingData != nil { - logger.Infof("📈 Backtest: Price ranking data ready for %d durations", - len(ctx.PriceRankingData.Durations)) - } - } - - record := &store.DecisionRecord{ - AccountState: store.AccountSnapshot{ - TotalBalance: accountInfo.TotalEquity, - AvailableBalance: accountInfo.AvailableBalance, - TotalUnrealizedProfit: unrealized, - PositionCount: accountInfo.PositionCount, - MarginUsedPct: accountInfo.MarginUsedPct, - }, - CandidateCoins: make([]string, 0, len(candidateCoins)), - Positions: r.snapshotPositions(priceMap), - } - for _, coin := range candidateCoins { - record.CandidateCoins = append(record.CandidateCoins, coin.Symbol) - } - record.Timestamp = time.UnixMilli(ts).UTC() - - return ctx, record, nil -} - -func (r *Runner) fillDecisionRecord(record *store.DecisionRecord, full *kernel.FullDecision) { - record.InputPrompt = full.UserPrompt - record.CoTTrace = full.CoTTrace - if len(full.Decisions) > 0 { - if data, err := json.MarshalIndent(full.Decisions, "", " "); err == nil { - record.DecisionJSON = string(data) - } - } -} - -func (r *Runner) invokeAIWithRetry(ctx *kernel.Context) (*kernel.FullDecision, error) { - var lastErr error - for attempt := 0; attempt < aiDecisionMaxRetries; attempt++ { - // Use GetFullDecisionWithStrategy with the pre-configured strategy engine - // This ensures backtest uses the same unified prompt generation as live trading - fd, err := kernel.GetFullDecisionWithStrategy( - ctx, - r.mcpClient, - r.strategyEngine, - r.cfg.PromptVariant, - ) - if err == nil { - return fd, nil - } - lastErr = err - delay := time.Duration(attempt+1) * 500 * time.Millisecond - time.Sleep(delay) - } - return nil, lastErr -} - -func (r *Runner) shouldTriggerDecision(barIndex int) bool { - if r.cfg.DecisionCadenceNBars <= 1 { - return true - } - if barIndex < 0 { - return true - } - return barIndex%r.cfg.DecisionCadenceNBars == 0 -} - -func (r *Runner) updateState(ts int64, equity, unrealized, marginUsed float64, priceMap map[string]float64, advancedDecision bool) { - r.stateMu.Lock() - defer r.stateMu.Unlock() - - if r.state.MaxEquity == 0 || equity > r.state.MaxEquity { - r.state.MaxEquity = equity - } - if r.state.MinEquity == 0 || equity < r.state.MinEquity { - r.state.MinEquity = equity - } - if r.state.MaxEquity > 0 { - drawdown := ((r.state.MaxEquity - equity) / r.state.MaxEquity) * 100 - if drawdown > r.state.MaxDrawdownPct { - r.state.MaxDrawdownPct = drawdown - } - } - - positions := make(map[string]PositionSnapshot) - for _, pos := range r.account.Positions() { - key := fmt.Sprintf("%s:%s", pos.Symbol, pos.Side) - positions[key] = PositionSnapshot{ - Symbol: pos.Symbol, - Side: pos.Side, - Quantity: pos.Quantity, - AvgPrice: pos.EntryPrice, - Leverage: pos.Leverage, - LiquidationPrice: pos.LiquidationPrice, - MarginUsed: pos.Margin, - OpenTime: pos.OpenTime, - AccumulatedFee: pos.AccumulatedFee, - } - } - - r.state.BarTimestamp = ts - r.state.BarIndex++ - if advancedDecision { - r.state.DecisionCycle++ - } - r.state.Cash = r.account.Cash() - r.state.Equity = equity - r.state.UnrealizedPnL = unrealized - r.state.RealizedPnL = r.account.RealizedPnL() - r.state.Positions = positions - r.state.LastUpdate = time.Now().UTC() -} - -func (r *Runner) handleStop(reason error) { - r.forceCheckpoint() - if reason != nil { - r.setLastError(reason) - } else { - r.setLastError(nil) - } - r.statusMu.Lock() - r.err = reason - r.status = RunStateStopped - r.statusMu.Unlock() - r.persistMetadata() - r.persistMetrics(true) - r.releaseLock() -} - -func (r *Runner) handlePause() { - r.forceCheckpoint() - r.setLastError(nil) - r.statusMu.Lock() - r.status = RunStatePaused - r.statusMu.Unlock() - r.persistMetadata() - r.persistMetrics(true) -} - -func (r *Runner) resumeFromPause() { - r.setLastError(nil) - r.statusMu.Lock() - r.status = RunStateRunning - r.statusMu.Unlock() - r.persistMetadata() -} - -func (r *Runner) handleCompletion() { - r.setLastError(nil) - r.statusMu.Lock() - r.status = RunStateCompleted - r.statusMu.Unlock() - r.persistMetadata() - r.persistMetrics(true) - r.releaseLock() -} - -func (r *Runner) handleFailure(err error) { - r.forceCheckpoint() - if err != nil { - r.setLastError(err) - } - r.statusMu.Lock() - r.err = err - r.status = RunStateFailed - r.statusMu.Unlock() - r.persistMetadata() - r.persistMetrics(true) - r.releaseLock() -} - -func (r *Runner) handleLiquidation() { - r.forceCheckpoint() - r.setLastError(errLiquidated) - r.statusMu.Lock() - r.err = errLiquidated - r.status = RunStateLiquidated - r.statusMu.Unlock() - r.persistMetadata() - r.persistMetrics(true) - r.releaseLock() -} - -func sortDecisionsByPriority(decisions []kernel.Decision) []kernel.Decision { - if len(decisions) <= 1 { - return decisions - } - - priority := func(action string) int { - switch action { - case "close_long", "close_short": - return 1 - case "open_long", "open_short": - return 2 - case "hold", "wait": - return 3 - default: - return 99 - } - } - - result := make([]kernel.Decision, len(decisions)) - copy(result, decisions) - - sort.Slice(result, func(i, j int) bool { - pi := priority(result[i].Action) - pj := priority(result[j].Action) - if pi != pj { - return pi < pj - } - return i < j - }) - - return result -} diff --git a/backtest/runner_metrics.go b/backtest/runner_metrics.go deleted file mode 100644 index e3dea04e..00000000 --- a/backtest/runner_metrics.go +++ /dev/null @@ -1,239 +0,0 @@ -package backtest - -import ( - "fmt" - "sort" - "time" - - "nofx/logger" - "nofx/store" -) - -func (r *Runner) persistMetadata() { - state := r.snapshotState() - meta := r.buildMetadata(state, r.Status()) - meta.CreatedAt = r.createdAt - if err := SaveRunMetadata(meta); err != nil { - logger.Infof("failed to save run metadata for %s: %v", r.cfg.RunID, err) - } else { - if err := updateRunIndex(meta, &r.cfg); err != nil { - logger.Infof("failed to update index for %s: %v", r.cfg.RunID, err) - } - } -} - -func (r *Runner) logDecision(record *store.DecisionRecord) error { - if record == nil { - return nil - } - persistDecisionRecord(r.cfg.RunID, record) - return nil -} - -func (r *Runner) persistMetrics(force bool) { - if r.cfg.RunID == "" { - return - } - - if !force && !r.lastMetricsWrite.IsZero() { - if time.Since(r.lastMetricsWrite) < metricsWriteInterval { - return - } - } - - state := r.snapshotState() - metrics, err := CalculateMetrics(r.cfg.RunID, &r.cfg, &state) - if err != nil { - logger.Infof("failed to compute metrics for %s: %v", r.cfg.RunID, err) - return - } - if metrics == nil { - return - } - if err := PersistMetrics(r.cfg.RunID, metrics); err != nil { - logger.Infof("failed to persist metrics for %s: %v", r.cfg.RunID, err) - return - } - r.lastMetricsWrite = time.Now() -} - -func (r *Runner) buildMetadata(state BacktestState, runState RunState) *RunMetadata { - if state.Liquidated && runState != RunStateLiquidated { - runState = RunStateLiquidated - } - - progress := progressPercent(state, r.cfg) - - summary := RunSummary{ - SymbolCount: len(r.cfg.Symbols), - DecisionTF: r.cfg.DecisionTimeframe, - ProcessedBars: state.BarIndex, - ProgressPct: progress, - EquityLast: state.Equity, - MaxDrawdownPct: state.MaxDrawdownPct, - Liquidated: state.Liquidated, - LiquidationNote: state.LiquidationNote, - } - - meta := &RunMetadata{ - RunID: r.cfg.RunID, - UserID: r.cfg.UserID, - State: runState, - LastError: r.lastErrorString(), - Summary: summary, - } - - return meta -} - -func progressPercent(state BacktestState, cfg BacktestConfig) float64 { - duration := cfg.Duration() - if duration <= 0 { - return 0 - } - if state.BarTimestamp == 0 { - return 0 - } - - start := time.Unix(cfg.StartTS, 0) - end := time.Unix(cfg.EndTS, 0) - current := time.UnixMilli(state.BarTimestamp) - - if !current.After(start) { - return 0 - } - if current.After(end) { - return 100 - } - - elapsed := current.Sub(start) - pct := float64(elapsed) / float64(duration) * 100 - if pct > 100 { - pct = 100 - } - if pct < 0 { - pct = 0 - } - return pct -} - -func (r *Runner) maybeCheckpoint() error { - state := r.snapshotState() - shouldCheckpoint := false - - if r.cfg.CheckpointIntervalBars > 0 && state.BarIndex > 0 && state.BarIndex%r.cfg.CheckpointIntervalBars == 0 { - shouldCheckpoint = true - } - - interval := time.Duration(r.cfg.CheckpointIntervalSeconds) * time.Second - if interval <= 0 { - interval = 2 * time.Second - } - if time.Since(r.lastCheckpoint) >= interval { - shouldCheckpoint = true - } - - if !shouldCheckpoint { - return nil - } - - if err := r.saveCheckpoint(state); err != nil { - return err - } - - return nil -} - -func (r *Runner) snapshotForCheckpoint(state BacktestState) []PositionSnapshot { - res := make([]PositionSnapshot, 0, len(state.Positions)) - for _, pos := range state.Positions { - res = append(res, pos) - } - sort.Slice(res, func(i, j int) bool { - if res[i].Symbol == res[j].Symbol { - return res[i].Side < res[j].Side - } - return res[i].Symbol < res[j].Symbol - }) - return res -} - -func (r *Runner) buildCheckpointFromState(state BacktestState) *Checkpoint { - return &Checkpoint{ - BarIndex: state.BarIndex, - BarTimestamp: state.BarTimestamp, - Cash: state.Cash, - Equity: state.Equity, - UnrealizedPnL: state.UnrealizedPnL, - RealizedPnL: state.RealizedPnL, - Positions: r.snapshotForCheckpoint(state), - DecisionCycle: state.DecisionCycle, - Liquidated: state.Liquidated, - LiquidationNote: state.LiquidationNote, - MaxEquity: state.MaxEquity, - MinEquity: state.MinEquity, - MaxDrawdownPct: state.MaxDrawdownPct, - AICacheRef: r.cachePath, - } -} - -func (r *Runner) saveCheckpoint(state BacktestState) error { - ckpt := r.buildCheckpointFromState(state) - if ckpt == nil { - return nil - } - if err := SaveCheckpoint(r.cfg.RunID, ckpt); err != nil { - return err - } - r.lastCheckpoint = time.Now() - return nil -} - -func (r *Runner) forceCheckpoint() { - state := r.snapshotState() - if err := r.saveCheckpoint(state); err != nil { - logger.Infof("failed to save checkpoint for %s: %v", r.cfg.RunID, err) - } -} - -func (r *Runner) RestoreFromCheckpoint() error { - ckpt, err := LoadCheckpoint(r.cfg.RunID) - if err != nil { - return err - } - return r.applyCheckpoint(ckpt) -} - -func (r *Runner) applyCheckpoint(ckpt *Checkpoint) error { - if ckpt == nil { - return fmt.Errorf("checkpoint is nil") - } - r.account.RestoreFromSnapshots(ckpt.Cash, ckpt.RealizedPnL, ckpt.Positions) - r.stateMu.Lock() - defer r.stateMu.Unlock() - r.state.BarIndex = ckpt.BarIndex - r.state.BarTimestamp = ckpt.BarTimestamp - r.state.Cash = ckpt.Cash - r.state.Equity = ckpt.Equity - r.state.UnrealizedPnL = ckpt.UnrealizedPnL - r.state.RealizedPnL = ckpt.RealizedPnL - r.state.DecisionCycle = ckpt.DecisionCycle - r.state.Liquidated = ckpt.Liquidated - r.state.LiquidationNote = ckpt.LiquidationNote - r.state.MaxEquity = ckpt.MaxEquity - r.state.MinEquity = ckpt.MinEquity - r.state.MaxDrawdownPct = ckpt.MaxDrawdownPct - r.state.Positions = snapshotsToMap(ckpt.Positions) - r.state.LastUpdate = time.Now().UTC() - r.lastCheckpoint = time.Now() - return nil -} - -func snapshotsToMap(snaps []PositionSnapshot) map[string]PositionSnapshot { - positions := make(map[string]PositionSnapshot, len(snaps)) - for _, snap := range snaps { - key := fmt.Sprintf("%s:%s", snap.Symbol, snap.Side) - positions[key] = snap - } - return positions -} diff --git a/backtest/runner_orders.go b/backtest/runner_orders.go deleted file mode 100644 index 8e6bdfa0..00000000 --- a/backtest/runner_orders.go +++ /dev/null @@ -1,420 +0,0 @@ -package backtest - -import ( - "fmt" - "strings" - "time" - - "nofx/kernel" - "nofx/logger" - "nofx/market" - "nofx/store" -) - -func (r *Runner) executeDecision(dec kernel.Decision, priceMap map[string]float64, ts int64, cycle int) (store.DecisionAction, []TradeEvent, string, error) { - symbol := dec.Symbol - if symbol == "" { - return store.DecisionAction{}, nil, "", fmt.Errorf("empty symbol in decision") - } - - usedLeverage := r.resolveLeverage(dec.Leverage, symbol) - actionRecord := store.DecisionAction{ - Action: dec.Action, - Symbol: symbol, - Leverage: usedLeverage, - Timestamp: time.UnixMilli(ts).UTC(), - } - - if priceMap == nil { - return actionRecord, nil, "", fmt.Errorf("priceMap is nil") - } - - basePrice, ok := priceMap[symbol] - if !ok || basePrice <= 0 { - return actionRecord, nil, "", fmt.Errorf("price unavailable for %s (found=%v, price=%.4f)", symbol, ok, basePrice) - } - fillPrice := r.executionPrice(symbol, basePrice, ts) - - switch dec.Action { - case "open_long": - qty := r.determineQuantity(dec, basePrice) - if qty <= 0 { - return actionRecord, nil, "", fmt.Errorf("invalid qty") - } - pos, fee, execPrice, err := r.account.Open(symbol, "long", qty, usedLeverage, fillPrice, ts) - if err != nil { - return actionRecord, nil, "", err - } - actionRecord.Quantity = qty - actionRecord.Price = execPrice - actionRecord.Leverage = pos.Leverage - trade := TradeEvent{ - Timestamp: ts, - Symbol: symbol, - Action: dec.Action, - Side: "long", - Quantity: qty, - Price: execPrice, - Fee: fee, - Slippage: execPrice - basePrice, - OrderValue: execPrice * qty, - RealizedPnL: 0, - Leverage: pos.Leverage, - Cycle: cycle, - PositionAfter: pos.Quantity, - } - return actionRecord, []TradeEvent{trade}, "", nil - - case "open_short": - qty := r.determineQuantity(dec, basePrice) - if qty <= 0 { - return actionRecord, nil, "", fmt.Errorf("invalid qty") - } - pos, fee, execPrice, err := r.account.Open(symbol, "short", qty, usedLeverage, fillPrice, ts) - if err != nil { - return actionRecord, nil, "", err - } - actionRecord.Quantity = qty - actionRecord.Price = execPrice - actionRecord.Leverage = pos.Leverage - trade := TradeEvent{ - Timestamp: ts, - Symbol: symbol, - Action: dec.Action, - Side: "short", - Quantity: qty, - Price: execPrice, - Fee: fee, - Slippage: basePrice - execPrice, - OrderValue: execPrice * qty, - RealizedPnL: 0, - Leverage: pos.Leverage, - Cycle: cycle, - PositionAfter: pos.Quantity, - } - return actionRecord, []TradeEvent{trade}, "", nil - - case "close_long": - qty := r.determineCloseQuantity(symbol, "long", dec) - if qty <= 0 { - return actionRecord, nil, "", fmt.Errorf("invalid close qty") - } - posLev := r.account.positionLeverage(symbol, "long") - realized, fee, execPrice, err := r.account.Close(symbol, "long", qty, fillPrice) - if err != nil { - return actionRecord, nil, "", err - } - actionRecord.Quantity = qty - actionRecord.Price = execPrice - actionRecord.Leverage = posLev - trade := TradeEvent{ - Timestamp: ts, - Symbol: symbol, - Action: dec.Action, - Side: "long", - Quantity: qty, - Price: execPrice, - Fee: fee, - Slippage: basePrice - execPrice, - OrderValue: execPrice * qty, - RealizedPnL: realized - fee, - Leverage: posLev, - Cycle: cycle, - PositionAfter: r.remainingPosition(symbol, "long"), - } - return actionRecord, []TradeEvent{trade}, "", nil - - case "close_short": - qty := r.determineCloseQuantity(symbol, "short", dec) - if qty <= 0 { - return actionRecord, nil, "", fmt.Errorf("invalid close qty") - } - posLev := r.account.positionLeverage(symbol, "short") - realized, fee, execPrice, err := r.account.Close(symbol, "short", qty, fillPrice) - if err != nil { - return actionRecord, nil, "", err - } - actionRecord.Quantity = qty - actionRecord.Price = execPrice - actionRecord.Leverage = posLev - trade := TradeEvent{ - Timestamp: ts, - Symbol: symbol, - Action: dec.Action, - Side: "short", - Quantity: qty, - Price: execPrice, - Fee: fee, - Slippage: execPrice - basePrice, - OrderValue: execPrice * qty, - RealizedPnL: realized - fee, - Leverage: posLev, - Cycle: cycle, - PositionAfter: r.remainingPosition(symbol, "short"), - } - return actionRecord, []TradeEvent{trade}, "", nil - - case "hold", "wait": - return actionRecord, nil, fmt.Sprintf("hold position: %s", dec.Action), nil - default: - return actionRecord, nil, "", fmt.Errorf("unsupported action %s", dec.Action) - } -} - -// MinPositionSizeUSD is the minimum position size in USD to avoid dust positions -const MinPositionSizeUSD = 10.0 - -func (r *Runner) determineQuantity(dec kernel.Decision, price float64) float64 { - snapshot := r.snapshotState() - equity := snapshot.Equity - if equity <= 0 { - equity = r.account.InitialBalance() - } - - // Get leverage for this symbol - leverage := r.resolveLeverage(dec.Leverage, dec.Symbol) - if leverage <= 0 { - leverage = 5 - } - - // Calculate available margin (leave some buffer for fees) - availableCash := r.account.Cash() - maxMarginToUse := availableCash * 0.9 // Use max 90% of available cash - maxPositionValue := maxMarginToUse * float64(leverage) - - sizeUSD := dec.PositionSizeUSD - if sizeUSD <= 0 { - // Default to 5% of equity, but cap to available margin - sizeUSD = 0.05 * equity - } - - // Cap position size to what we can actually afford - if sizeUSD > maxPositionValue { - logger.Infof("📊 Backtest: capping position from %.2f to %.2f (available margin: %.2f, leverage: %dx)", - sizeUSD, maxPositionValue, maxMarginToUse, leverage) - sizeUSD = maxPositionValue - } - - // Reject positions below minimum size to avoid dust positions - if sizeUSD < MinPositionSizeUSD { - logger.Infof("📊 Backtest: rejecting position size %.2f USD (below minimum %.2f USD)", - sizeUSD, MinPositionSizeUSD) - return 0 - } - - qty := sizeUSD / price - if qty < 0 { - qty = 0 - } - return qty -} - -func (r *Runner) determineCloseQuantity(symbol, side string, dec kernel.Decision) float64 { - for _, pos := range r.account.Positions() { - if pos.Symbol == strings.ToUpper(symbol) && pos.Side == side { - return pos.Quantity - } - } - return 0 -} - -func (r *Runner) resolveLeverage(requested int, symbol string) int { - sym := strings.ToUpper(symbol) - isBTCETH := sym == "BTCUSDT" || sym == "ETHUSDT" - - // Determine configured max leverage for this symbol type - var maxLeverage int - if isBTCETH { - maxLeverage = r.cfg.Leverage.BTCETHLeverage - if maxLeverage <= 0 { - maxLeverage = 10 // Default max for BTC/ETH - } - } else { - maxLeverage = r.cfg.Leverage.AltcoinLeverage - if maxLeverage <= 0 { - maxLeverage = 5 // Default max for altcoins - } - } - - // Use requested leverage if provided, otherwise use max as default - leverage := requested - if leverage <= 0 { - leverage = maxLeverage - } - - // Enforce max leverage limit - if leverage > maxLeverage { - logger.Infof("📊 Backtest: capping leverage from %dx to %dx for %s", - leverage, maxLeverage, symbol) - leverage = maxLeverage - } - - return leverage -} - -func (r *Runner) remainingPosition(symbol, side string) float64 { - for _, pos := range r.account.Positions() { - if pos.Symbol == strings.ToUpper(symbol) && pos.Side == side { - return pos.Quantity - } - } - return 0 -} - -func (r *Runner) snapshotPositions(priceMap map[string]float64) []store.PositionSnapshot { - positions := r.account.Positions() - list := make([]store.PositionSnapshot, 0, len(positions)) - for _, pos := range positions { - price := priceMap[pos.Symbol] - list = append(list, store.PositionSnapshot{ - Symbol: pos.Symbol, - Side: pos.Side, - PositionAmt: pos.Quantity, - EntryPrice: pos.EntryPrice, - MarkPrice: price, - UnrealizedProfit: unrealizedPnL(pos, price), - Leverage: float64(pos.Leverage), - LiquidationPrice: pos.LiquidationPrice, - }) - } - return list -} - -func (r *Runner) convertPositions(priceMap map[string]float64) []kernel.PositionInfo { - positions := r.account.Positions() - list := make([]kernel.PositionInfo, 0, len(positions)) - for _, pos := range positions { - price := priceMap[pos.Symbol] - pnl := unrealizedPnL(pos, price) - // Calculate P&L percentage based on entry notional (position cost) - pnlPct := 0.0 - if pos.Notional > 0 { - pnlPct = (pnl / pos.Notional) * 100 - } - list = append(list, kernel.PositionInfo{ - Symbol: pos.Symbol, - Side: pos.Side, - EntryPrice: pos.EntryPrice, - MarkPrice: price, - Quantity: pos.Quantity, - Leverage: pos.Leverage, - UnrealizedPnL: pnl, - UnrealizedPnLPct: pnlPct, - LiquidationPrice: pos.LiquidationPrice, - MarginUsed: pos.Margin, - UpdateTime: time.Now().UnixMilli(), - }) - } - return list -} - -func (r *Runner) executionPrice(symbol string, markPrice float64, ts int64) float64 { - curr, next := r.feed.decisionBarSnapshot(symbol, ts) - switch r.cfg.FillPolicy { - case FillPolicyNextOpen: - if next != nil && next.Open > 0 { - return next.Open - } - case FillPolicyBarVWAP: - if curr != nil { - if vwap := barVWAP(*curr); vwap > 0 { - return vwap - } - } - case FillPolicyMidPrice: - if curr != nil && curr.High > 0 && curr.Low > 0 { - return (curr.High + curr.Low) / 2 - } - } - return markPrice -} - -func (r *Runner) totalMarginUsed() float64 { - sum := 0.0 - for _, pos := range r.account.Positions() { - sum += pos.Margin - } - return sum -} - -func (r *Runner) checkLiquidation(ts int64, priceMap map[string]float64, cycle int) ([]TradeEvent, string, error) { - positions := append([]*position(nil), r.account.Positions()...) - events := make([]TradeEvent, 0) - var noteBuilder strings.Builder - - for _, pos := range positions { - price := priceMap[pos.Symbol] - liqPrice := pos.LiquidationPrice - trigger := false - execPrice := price - if pos.Side == "long" { - if price <= liqPrice && liqPrice > 0 { - trigger = true - execPrice = liqPrice - } - } else { - if price >= liqPrice && liqPrice > 0 { - trigger = true - execPrice = liqPrice - } - } - if !trigger { - continue - } - - realized, fee, finalPrice, err := r.account.Close(pos.Symbol, pos.Side, pos.Quantity, execPrice) - if err != nil { - return nil, "", err - } - - noteBuilder.WriteString(fmt.Sprintf("%s %s @ %.4f; ", pos.Symbol, pos.Side, finalPrice)) - - evt := TradeEvent{ - Timestamp: ts, - Symbol: pos.Symbol, - Action: "liquidated", - Side: pos.Side, - Quantity: pos.Quantity, - Price: finalPrice, - Fee: fee, - Slippage: 0, - OrderValue: finalPrice * pos.Quantity, - RealizedPnL: realized - fee, - Leverage: pos.Leverage, - Cycle: cycle, - PositionAfter: 0, - LiquidationFlag: true, - Note: fmt.Sprintf("forced liquidation at %.4f", finalPrice), - } - events = append(events, evt) - } - - if len(events) == 0 { - return events, "", nil - } - - note := strings.TrimSuffix(noteBuilder.String(), "; ") - - r.stateMu.Lock() - r.state.Liquidated = true - r.state.LiquidationNote = note - r.stateMu.Unlock() - - return events, note, nil -} - -func barVWAP(k market.Kline) float64 { - values := []float64{k.Open, k.High, k.Low, k.Close} - sum := 0.0 - count := 0.0 - for _, v := range values { - if v > 0 { - sum += v - count++ - } - } - if count == 0 { - return 0 - } - return sum / count -} diff --git a/backtest/storage.go b/backtest/storage.go deleted file mode 100644 index 84cbcaa6..00000000 --- a/backtest/storage.go +++ /dev/null @@ -1,561 +0,0 @@ -package backtest - -import ( - "archive/zip" - "bufio" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "nofx/store" -) - -const ( - backtestsRootDir = "backtests" -) - -type progressPayload struct { - BarIndex int `json:"bar_index"` - Equity float64 `json:"equity"` - ProgressPct float64 `json:"progress_pct"` - Liquidated bool `json:"liquidated"` - UpdatedAtISO string `json:"updated_at_iso"` -} - -func runDir(runID string) string { - return filepath.Join(backtestsRootDir, runID) -} - -func ensureRunDir(runID string) error { - dir := runDir(runID) - return os.MkdirAll(dir, 0o755) -} - -func checkpointPath(runID string) string { - return filepath.Join(runDir(runID), "checkpoint.json") -} - -func runMetadataPath(runID string) string { - return filepath.Join(runDir(runID), "run.json") -} - -func equityLogPath(runID string) string { - return filepath.Join(runDir(runID), "equity.jsonl") -} - -func tradesLogPath(runID string) string { - return filepath.Join(runDir(runID), "trades.jsonl") -} - -func metricsPath(runID string) string { - return filepath.Join(runDir(runID), "metrics.json") -} - -func progressPath(runID string) string { - return filepath.Join(runDir(runID), "progress.json") -} - -func decisionLogDir(runID string) string { - return filepath.Join(runDir(runID), "decision_logs") -} - -func writeJSONAtomic(path string, v any) error { - data, err := json.MarshalIndent(v, "", " ") - if err != nil { - return err - } - return writeFileAtomic(path, data, 0o644) -} - -func writeFileAtomic(path string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - tmpFile, err := os.CreateTemp(dir, ".tmp-*") - if err != nil { - return err - } - tmpPath := tmpFile.Name() - if _, err := tmpFile.Write(data); err != nil { - tmpFile.Close() - os.Remove(tmpPath) - return err - } - if err := tmpFile.Sync(); err != nil { - tmpFile.Close() - os.Remove(tmpPath) - return err - } - if err := tmpFile.Close(); err != nil { - os.Remove(tmpPath) - return err - } - if err := os.Chmod(tmpPath, perm); err != nil { - os.Remove(tmpPath) - return err - } - return os.Rename(tmpPath, path) -} - -func appendJSONLine(path string, payload any) error { - data, err := json.Marshal(payload) - if err != nil { - return err - } - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - return err - } - defer f.Close() - - writer := bufio.NewWriter(f) - if _, err := writer.Write(data); err != nil { - return err - } - if err := writer.WriteByte('\n'); err != nil { - return err - } - if err := writer.Flush(); err != nil { - return err - } - return f.Sync() -} - -// SaveCheckpoint writes the checkpoint to disk. -func SaveCheckpoint(runID string, ckpt *Checkpoint) error { - if ckpt == nil { - return fmt.Errorf("checkpoint is nil") - } - if usingDB() { - return saveCheckpointDB(runID, ckpt) - } - return writeJSONAtomic(checkpointPath(runID), ckpt) -} - -// LoadCheckpoint reads the most recent checkpoint. -func LoadCheckpoint(runID string) (*Checkpoint, error) { - if usingDB() { - return loadCheckpointDB(runID) - } - path := checkpointPath(runID) - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var ckpt Checkpoint - if err := json.Unmarshal(data, &ckpt); err != nil { - return nil, err - } - return &ckpt, nil -} - -// SaveRunMetadata writes to run.json. -func SaveRunMetadata(meta *RunMetadata) error { - if meta == nil { - return fmt.Errorf("run metadata is nil") - } - if meta.Version == 0 { - meta.Version = 1 - } - if meta.CreatedAt.IsZero() { - meta.CreatedAt = time.Now().UTC() - } - meta.UpdatedAt = time.Now().UTC() - if usingDB() { - return saveRunMetadataDB(meta) - } - return writeJSONAtomic(runMetadataPath(meta.RunID), meta) -} - -// LoadRunMetadata reads run.json. -func LoadRunMetadata(runID string) (*RunMetadata, error) { - if usingDB() { - return loadRunMetadataDB(runID) - } - path := runMetadataPath(runID) - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var meta RunMetadata - if err := json.Unmarshal(data, &meta); err != nil { - return nil, err - } - return &meta, nil -} - -func appendEquityPoint(runID string, point EquityPoint) error { - if usingDB() { - return appendEquityPointDB(runID, point) - } - return appendJSONLine(equityLogPath(runID), point) -} - -func appendTradeEvent(runID string, event TradeEvent) error { - if usingDB() { - return appendTradeEventDB(runID, event) - } - return appendJSONLine(tradesLogPath(runID), event) -} - -func saveMetrics(runID string, metrics *Metrics) error { - if metrics == nil { - return fmt.Errorf("metrics is nil") - } - if usingDB() { - return saveMetricsDB(runID, metrics) - } - return writeJSONAtomic(metricsPath(runID), metrics) -} - -func saveProgress(runID string, state *BacktestState, cfg *BacktestConfig) error { - if state == nil || cfg == nil { - return fmt.Errorf("state or config nil") - } - dur := cfg.Duration() - progress := 0.0 - if dur > 0 { - current := time.UnixMilli(state.BarTimestamp) - start := time.Unix(cfg.StartTS, 0) - if current.After(start) { - elapsed := current.Sub(start) - progress = float64(elapsed) / float64(dur) - } - } - payload := progressPayload{ - BarIndex: state.BarIndex, - Equity: state.Equity, - ProgressPct: progress * 100, - Liquidated: state.Liquidated, - - UpdatedAtISO: time.Now().UTC().Format(time.RFC3339), - } - if usingDB() { - return saveProgressDB(runID, payload) - } - return writeJSONAtomic(progressPath(runID), payload) -} - -func SaveConfig(runID string, cfg *BacktestConfig) error { - if cfg == nil { - return fmt.Errorf("config is nil") - } - persist := *cfg - persist.AICfg.APIKey = "" - if usingDB() { - return saveConfigDB(runID, &persist) - } - if err := ensureRunDir(runID); err != nil { - return err - } - return writeJSONAtomic(filepath.Join(runDir(runID), "config.json"), &persist) -} - -func LoadConfig(runID string) (*BacktestConfig, error) { - if usingDB() { - return loadConfigDB(runID) - } - data, err := os.ReadFile(filepath.Join(runDir(runID), "config.json")) - if err != nil { - return nil, err - } - var cfg BacktestConfig - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -func LoadEquityPoints(runID string) ([]EquityPoint, error) { - if usingDB() { - return loadEquityPointsDB(runID) - } - points, err := loadJSONLines[EquityPoint](equityLogPath(runID)) - if err != nil { - return nil, err - } - sort.Slice(points, func(i, j int) bool { - return points[i].Timestamp < points[j].Timestamp - }) - return points, nil -} - -func LoadTradeEvents(runID string) ([]TradeEvent, error) { - if usingDB() { - return loadTradeEventsDB(runID) - } - events, err := loadJSONLines[TradeEvent](tradesLogPath(runID)) - if err != nil { - return nil, err - } - sort.Slice(events, func(i, j int) bool { - if events[i].Timestamp == events[j].Timestamp { - return events[i].Symbol < events[j].Symbol - } - return events[i].Timestamp < events[j].Timestamp - }) - return events, nil -} - -func LoadMetrics(runID string) (*Metrics, error) { - if usingDB() { - return loadMetricsDB(runID) - } - data, err := os.ReadFile(metricsPath(runID)) - if err != nil { - return nil, err - } - var metrics Metrics - if err := json.Unmarshal(data, &metrics); err != nil { - return nil, err - } - return &metrics, nil -} - -func LoadRunIDs() ([]string, error) { - if usingDB() { - return loadRunIDsDB() - } - entries, err := os.ReadDir(backtestsRootDir) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return []string{}, nil - } - return nil, err - } - runIDs := make([]string, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - runIDs = append(runIDs, entry.Name()) - } - } - sort.Strings(runIDs) - return runIDs, nil -} - -func loadJSONLines[T any](path string) ([]T, error) { - file, err := os.Open(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return []T{}, nil - } - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) - - var result []T - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - var item T - if err := json.Unmarshal(line, &item); err != nil { - return nil, err - } - result = append(result, item) - } - - if err := scanner.Err(); err != nil { - return nil, err - } - - return result, nil -} -func PersistMetrics(runID string, metrics *Metrics) error { - return saveMetrics(runID, metrics) -} - -func LoadDecisionTrace(runID string, cycle int) (*store.DecisionRecord, error) { - if usingDB() { - return loadDecisionTraceDB(runID, cycle) - } - dir := decisionLogDir(runID) - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - type candidate struct { - path string - info os.DirEntry - } - cands := make([]candidate, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - name := entry.Name() - if !strings.HasPrefix(name, "decision_") || !strings.HasSuffix(name, ".json") { - continue - } - cands = append(cands, candidate{path: filepath.Join(dir, name), info: entry}) - } - sort.Slice(cands, func(i, j int) bool { - infoI, _ := cands[i].info.Info() - infoJ, _ := cands[j].info.Info() - if infoI == nil || infoJ == nil { - return cands[i].path > cands[j].path - } - return infoI.ModTime().After(infoJ.ModTime()) - }) - - for _, cand := range cands { - data, err := os.ReadFile(cand.path) - if err != nil { - continue - } - var record store.DecisionRecord - if err := json.Unmarshal(data, &record); err != nil { - continue - } - if cycle <= 0 || record.CycleNumber == cycle { - return &record, nil - } - } - return nil, fmt.Errorf("decision trace not found for run %s cycle %d", runID, cycle) -} - -func LoadDecisionRecords(runID string, limit, offset int) ([]*store.DecisionRecord, error) { - if limit <= 0 { - limit = 20 - } - if offset < 0 { - offset = 0 - } - if usingDB() { - return loadDecisionRecordsDB(runID, limit, offset) - } - dir := decisionLogDir(runID) - entries, err := os.ReadDir(dir) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return []*store.DecisionRecord{}, nil - } - return nil, err - } - type fileEntry struct { - path string - info os.DirEntry - } - files := make([]fileEntry, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - name := entry.Name() - if !strings.HasPrefix(name, "decision_") || !strings.HasSuffix(name, ".json") { - continue - } - files = append(files, fileEntry{path: filepath.Join(dir, name), info: entry}) - } - sort.Slice(files, func(i, j int) bool { - infoI, _ := files[i].info.Info() - infoJ, _ := files[j].info.Info() - if infoI == nil || infoJ == nil { - return files[i].path > files[j].path - } - return infoI.ModTime().After(infoJ.ModTime()) - }) - if offset >= len(files) { - return []*store.DecisionRecord{}, nil - } - end := offset + limit - if end > len(files) { - end = len(files) - } - records := make([]*store.DecisionRecord, 0, end-offset) - for _, file := range files[offset:end] { - data, err := os.ReadFile(file.path) - if err != nil { - continue - } - var record store.DecisionRecord - if err := json.Unmarshal(data, &record); err != nil { - continue - } - records = append(records, &record) - } - return records, nil -} - -func CreateRunExport(runID string) (string, error) { - if usingDB() { - return createRunExportDB(runID) - } - root := runDir(runID) - if _, err := os.Stat(root); err != nil { - return "", err - } - tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s-*.zip", runID)) - if err != nil { - return "", err - } - defer tmpFile.Close() - - zipWriter := zip.NewWriter(tmpFile) - err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - info, err := d.Info() - if err != nil { - return err - } - header, err := zip.FileInfoHeader(info) - if err != nil { - return err - } - header.Name = rel - header.Method = zip.Deflate - writer, err := zipWriter.CreateHeader(header) - if err != nil { - return err - } - src, err := os.Open(path) - if err != nil { - return err - } - if _, err := io.Copy(writer, src); err != nil { - src.Close() - return err - } - src.Close() - return nil - }) - if err != nil { - zipWriter.Close() - return "", err - } - if err := zipWriter.Close(); err != nil { - return "", err - } - return tmpFile.Name(), nil -} - -func persistDecisionRecord(runID string, record *store.DecisionRecord) { - if !usingDB() || record == nil { - return - } - _ = saveDecisionRecordDB(runID, record) -} diff --git a/backtest/storage_db_impl.go b/backtest/storage_db_impl.go deleted file mode 100644 index 2eb2b407..00000000 --- a/backtest/storage_db_impl.go +++ /dev/null @@ -1,498 +0,0 @@ -package backtest - -import ( - "archive/zip" - "database/sql" - "encoding/json" - "errors" - "fmt" - "os" - "time" - - "nofx/store" -) - -func saveCheckpointDB(runID string, ckpt *Checkpoint) error { - data, err := json.Marshal(ckpt) - if err != nil { - return err - } - _, err = persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_checkpoints (run_id, payload, updated_at) - VALUES (?, ?, CURRENT_TIMESTAMP) - ON CONFLICT(run_id) DO UPDATE SET payload=excluded.payload, updated_at=CURRENT_TIMESTAMP - `), runID, data) - return err -} - -func loadCheckpointDB(runID string) (*Checkpoint, error) { - var payload []byte - err := persistenceDB.QueryRow(convertQuery(`SELECT payload FROM backtest_checkpoints WHERE run_id = ?`), runID).Scan(&payload) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, os.ErrNotExist - } - return nil, err - } - var ckpt Checkpoint - if err := json.Unmarshal(payload, &ckpt); err != nil { - return nil, err - } - return &ckpt, nil -} - -func saveConfigDB(runID string, cfg *BacktestConfig) error { - persist := *cfg - persist.AICfg.APIKey = "" - data, err := json.Marshal(&persist) - if err != nil { - return err - } - template := cfg.PromptTemplate - if template == "" { - template = "default" - } - now := time.Now().UTC().Format(time.RFC3339) - userID := cfg.UserID - if userID == "" { - userID = "default" - } - _, err = persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_runs (run_id, user_id, config_json, prompt_template, custom_prompt, override_prompt, ai_provider, ai_model, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id) DO NOTHING - `), runID, userID, data, template, cfg.CustomPrompt, cfg.OverrideBasePrompt, cfg.AICfg.Provider, cfg.AICfg.Model, now, now) - if err != nil { - return err - } - _, err = persistenceDB.Exec(convertQuery(` - UPDATE backtest_runs - SET user_id = ?, config_json = ?, prompt_template = ?, custom_prompt = ?, override_prompt = ?, ai_provider = ?, ai_model = ?, updated_at = CURRENT_TIMESTAMP - WHERE run_id = ? - `), userID, data, template, cfg.CustomPrompt, cfg.OverrideBasePrompt, cfg.AICfg.Provider, cfg.AICfg.Model, runID) - return err -} - -func loadConfigDB(runID string) (*BacktestConfig, error) { - var payload []byte - err := persistenceDB.QueryRow(convertQuery(`SELECT config_json FROM backtest_runs WHERE run_id = ?`), runID).Scan(&payload) - if err != nil { - return nil, err - } - if len(payload) == 0 { - return nil, fmt.Errorf("config missing for %s", runID) - } - var cfg BacktestConfig - if err := json.Unmarshal(payload, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -func saveRunMetadataDB(meta *RunMetadata) error { - created := meta.CreatedAt.UTC().Format(time.RFC3339) - updated := meta.UpdatedAt.UTC().Format(time.RFC3339) - userID := meta.UserID - if userID == "" { - userID = "default" - } - if _, err := persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_runs (run_id, user_id, label, last_error, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(run_id) DO NOTHING - `), meta.RunID, userID, meta.Label, meta.LastError, created, updated); err != nil { - return err - } - _, err := persistenceDB.Exec(convertQuery(` - UPDATE backtest_runs - SET user_id = ?, state = ?, symbol_count = ?, decision_tf = ?, processed_bars = ?, progress_pct = ?, equity_last = ?, max_drawdown_pct = ?, liquidated = ?, liquidation_note = ?, label = ?, last_error = ?, updated_at = ? - WHERE run_id = ? - `), userID, string(meta.State), meta.Summary.SymbolCount, meta.Summary.DecisionTF, meta.Summary.ProcessedBars, meta.Summary.ProgressPct, meta.Summary.EquityLast, meta.Summary.MaxDrawdownPct, meta.Summary.Liquidated, meta.Summary.LiquidationNote, meta.Label, meta.LastError, updated, meta.RunID) - return err -} - -func loadRunMetadataDB(runID string) (*RunMetadata, error) { - var ( - userID string - state string - label string - lastErr string - symbolCount int - decisionTF string - processedBars int - progressPct float64 - equityLast float64 - maxDD float64 - liquidated bool - liquidationNote string - createdISO string - updatedISO string - ) - err := persistenceDB.QueryRow(convertQuery(` - SELECT user_id, state, label, last_error, symbol_count, decision_tf, processed_bars, progress_pct, equity_last, max_drawdown_pct, liquidated, liquidation_note, created_at, updated_at - FROM backtest_runs WHERE run_id = ? - `), runID).Scan(&userID, &state, &label, &lastErr, &symbolCount, &decisionTF, &processedBars, &progressPct, &equityLast, &maxDD, &liquidated, &liquidationNote, &createdISO, &updatedISO) - if err != nil { - return nil, err - } - meta := &RunMetadata{ - RunID: runID, - UserID: userID, - Version: 1, - State: RunState(state), - Label: label, - LastError: lastErr, - Summary: RunSummary{ - SymbolCount: symbolCount, - DecisionTF: decisionTF, - ProcessedBars: processedBars, - ProgressPct: progressPct, - EquityLast: equityLast, - MaxDrawdownPct: maxDD, - Liquidated: liquidated, - LiquidationNote: liquidationNote, - }, - } - if meta.UserID == "" { - meta.UserID = "default" - } - if t, err := time.Parse(time.RFC3339, createdISO); err == nil { - meta.CreatedAt = t - } - if t, err := time.Parse(time.RFC3339, updatedISO); err == nil { - meta.UpdatedAt = t - } - return meta, nil -} - -func loadRunIDsDB() ([]string, error) { - rows, err := persistenceDB.Query(`SELECT run_id FROM backtest_runs ORDER BY updated_at DESC`) - if err != nil { - return nil, err - } - defer rows.Close() - var ids []string - for rows.Next() { - var runID string - if err := rows.Scan(&runID); err != nil { - return nil, err - } - ids = append(ids, runID) - } - return ids, rows.Err() -} - -func appendEquityPointDB(runID string, point EquityPoint) error { - _, err := persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_equity (run_id, ts, equity, available, pnl, pnl_pct, dd_pct, cycle) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `), runID, point.Timestamp, point.Equity, point.Available, point.PnL, point.PnLPct, point.DrawdownPct, point.Cycle) - return err -} - -func loadEquityPointsDB(runID string) ([]EquityPoint, error) { - rows, err := persistenceDB.Query(convertQuery(` - SELECT ts, equity, available, pnl, pnl_pct, dd_pct, cycle - FROM backtest_equity WHERE run_id = ? ORDER BY ts ASC - `), runID) - if err != nil { - return nil, err - } - defer rows.Close() - points := make([]EquityPoint, 0) - for rows.Next() { - var point EquityPoint - if err := rows.Scan(&point.Timestamp, &point.Equity, &point.Available, &point.PnL, &point.PnLPct, &point.DrawdownPct, &point.Cycle); err != nil { - return nil, err - } - points = append(points, point) - } - return points, rows.Err() -} - -func appendTradeEventDB(runID string, event TradeEvent) error { - _, err := persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_trades (run_id, ts, symbol, action, side, qty, price, fee, slippage, order_value, realized_pnl, leverage, cycle, position_after, liquidation, note) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), runID, event.Timestamp, event.Symbol, event.Action, event.Side, event.Quantity, event.Price, event.Fee, event.Slippage, event.OrderValue, event.RealizedPnL, event.Leverage, event.Cycle, event.PositionAfter, event.LiquidationFlag, event.Note) - return err -} - -func loadTradeEventsDB(runID string) ([]TradeEvent, error) { - rows, err := persistenceDB.Query(convertQuery(` - SELECT ts, symbol, action, side, qty, price, fee, slippage, order_value, realized_pnl, leverage, cycle, position_after, liquidation, note - FROM backtest_trades WHERE run_id = ? ORDER BY ts ASC - `), runID) - if err != nil { - return nil, err - } - defer rows.Close() - events := make([]TradeEvent, 0) - for rows.Next() { - var event TradeEvent - if err := rows.Scan(&event.Timestamp, &event.Symbol, &event.Action, &event.Side, &event.Quantity, &event.Price, &event.Fee, &event.Slippage, &event.OrderValue, &event.RealizedPnL, &event.Leverage, &event.Cycle, &event.PositionAfter, &event.LiquidationFlag, &event.Note); err != nil { - return nil, err - } - events = append(events, event) - } - return events, rows.Err() -} - -func saveMetricsDB(runID string, metrics *Metrics) error { - data, err := json.Marshal(metrics) - if err != nil { - return err - } - _, err = persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_metrics (run_id, payload, updated_at) - VALUES (?, ?, CURRENT_TIMESTAMP) - ON CONFLICT(run_id) DO UPDATE SET payload=excluded.payload, updated_at=CURRENT_TIMESTAMP - `), runID, data) - return err -} - -func loadMetricsDB(runID string) (*Metrics, error) { - var payload []byte - err := persistenceDB.QueryRow(convertQuery(`SELECT payload FROM backtest_metrics WHERE run_id = ?`), runID).Scan(&payload) - if err != nil { - return nil, err - } - var metrics Metrics - if err := json.Unmarshal(payload, &metrics); err != nil { - return nil, err - } - return &metrics, nil -} - -func saveProgressDB(runID string, payload progressPayload) error { - _, err := persistenceDB.Exec(convertQuery(` - UPDATE backtest_runs - SET progress_pct = ?, equity_last = ?, processed_bars = ?, liquidated = ?, updated_at = ? - WHERE run_id = ? - `), payload.ProgressPct, payload.Equity, payload.BarIndex, payload.Liquidated, payload.UpdatedAtISO, runID) - return err -} - -func loadDecisionTraceDB(runID string, cycle int) (*store.DecisionRecord, error) { - var rows *sql.Rows - var err error - if cycle > 0 { - rows, err = persistenceDB.Query(convertQuery(`SELECT payload FROM backtest_decisions WHERE run_id = ? AND cycle = ? ORDER BY created_at DESC LIMIT 1`), runID, cycle) - } else { - rows, err = persistenceDB.Query(convertQuery(`SELECT payload FROM backtest_decisions WHERE run_id = ? ORDER BY created_at DESC LIMIT 1`), runID) - } - if err != nil { - return nil, err - } - defer rows.Close() - if !rows.Next() { - return nil, fmt.Errorf("decision trace not found for %s", runID) - } - var payload []byte - if err := rows.Scan(&payload); err != nil { - return nil, err - } - var record store.DecisionRecord - if err := json.Unmarshal(payload, &record); err != nil { - return nil, err - } - return &record, nil -} - -func saveDecisionRecordDB(runID string, record *store.DecisionRecord) error { - if record == nil { - return nil - } - data, err := json.Marshal(record) - if err != nil { - return err - } - _, err = persistenceDB.Exec(convertQuery(` - INSERT INTO backtest_decisions (run_id, cycle, payload) - VALUES (?, ?, ?) - `), runID, record.CycleNumber, data) - return err -} - -func loadDecisionRecordsDB(runID string, limit, offset int) ([]*store.DecisionRecord, error) { - rows, err := persistenceDB.Query(convertQuery(` - SELECT payload FROM backtest_decisions - WHERE run_id = ? - ORDER BY id DESC - LIMIT ? OFFSET ? - `), runID, limit, offset) - if err != nil { - return nil, err - } - defer rows.Close() - records := make([]*store.DecisionRecord, 0, limit) - for rows.Next() { - var payload []byte - if err := rows.Scan(&payload); err != nil { - return nil, err - } - var record store.DecisionRecord - if err := json.Unmarshal(payload, &record); err != nil { - return nil, err - } - records = append(records, &record) - } - return records, rows.Err() -} - -func createRunExportDB(runID string) (string, error) { - tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s-*.zip", runID)) - if err != nil { - return "", err - } - defer tmpFile.Close() - - zipWriter := zip.NewWriter(tmpFile) - defer zipWriter.Close() - - if meta, err := loadRunMetadataDB(runID); err == nil { - if err := writeJSONToZip(zipWriter, "run.json", meta); err != nil { - return "", err - } - } - if cfg, err := loadConfigDB(runID); err == nil { - if err := writeJSONToZip(zipWriter, "config.json", cfg); err != nil { - return "", err - } - } - if ckpt, err := loadCheckpointDB(runID); err == nil { - if err := writeJSONToZip(zipWriter, "checkpoint.json", ckpt); err != nil { - return "", err - } - } - if metrics, err := loadMetricsDB(runID); err == nil { - if err := writeJSONToZip(zipWriter, "metrics.json", metrics); err != nil { - return "", err - } - } - if points, err := loadEquityPointsDB(runID); err == nil && len(points) > 0 { - if err := writeJSONLinesToZip(zipWriter, "equity.jsonl", points); err != nil { - return "", err - } - } - if trades, err := loadTradeEventsDB(runID); err == nil && len(trades) > 0 { - if err := writeJSONLinesToZip(zipWriter, "trades.jsonl", trades); err != nil { - return "", err - } - } - if err := writeDecisionLogsToZip(zipWriter, runID); err != nil { - return "", err - } - - if err := zipWriter.Close(); err != nil { - return "", err - } - if err := tmpFile.Sync(); err != nil { - return "", err - } - return tmpFile.Name(), nil -} - -func writeJSONToZip(z *zip.Writer, name string, value any) error { - data, err := json.MarshalIndent(value, "", " ") - if err != nil { - return err - } - w, err := z.Create(name) - if err != nil { - return err - } - _, err = w.Write(data) - return err -} - -func writeJSONLinesToZip[T any](z *zip.Writer, name string, items []T) error { - w, err := z.Create(name) - if err != nil { - return err - } - for _, item := range items { - data, err := json.Marshal(item) - if err != nil { - return err - } - if _, err := w.Write(data); err != nil { - return err - } - if _, err := w.Write([]byte("\n")); err != nil { - return err - } - } - return nil -} - -func writeDecisionLogsToZip(z *zip.Writer, runID string) error { - rows, err := persistenceDB.Query(convertQuery(` - SELECT id, cycle, payload FROM backtest_decisions - WHERE run_id = ? ORDER BY id ASC - `), runID) - if err != nil { - return err - } - defer rows.Close() - for rows.Next() { - var ( - id int64 - cycle int - payload []byte - ) - if err := rows.Scan(&id, &cycle, &payload); err != nil { - return err - } - name := fmt.Sprintf("decision_logs/decision_%d_cycle%d.json", id, cycle) - w, err := z.Create(name) - if err != nil { - return err - } - if _, err := w.Write(payload); err != nil { - return err - } - } - return rows.Err() -} - -func listIndexEntriesDB() ([]RunIndexEntry, error) { - rows, err := persistenceDB.Query(` - SELECT run_id, state, symbol_count, decision_tf, equity_last, max_drawdown_pct, created_at, updated_at, config_json - FROM backtest_runs - ORDER BY updated_at DESC - `) - if err != nil { - return nil, err - } - defer rows.Close() - var entries []RunIndexEntry - for rows.Next() { - var ( - entry RunIndexEntry - createdISO string - updatedISO string - cfgJSON []byte - symbolCnt int - ) - if err := rows.Scan(&entry.RunID, &entry.State, &symbolCnt, &entry.DecisionTF, &entry.EquityLast, &entry.MaxDrawdownPct, &createdISO, &updatedISO, &cfgJSON); err != nil { - return nil, err - } - entry.CreatedAtISO = createdISO - entry.UpdatedAtISO = updatedISO - entry.Symbols = make([]string, 0, symbolCnt) - var cfg BacktestConfig - if len(cfgJSON) > 0 && json.Unmarshal(cfgJSON, &cfg) == nil { - entry.Symbols = append([]string(nil), cfg.Symbols...) - entry.StartTS = cfg.StartTS - entry.EndTS = cfg.EndTS - } - entries = append(entries, entry) - } - return entries, rows.Err() -} - -func deleteRunDB(runID string) error { - _, err := persistenceDB.Exec(convertQuery(`DELETE FROM backtest_runs WHERE run_id = ?`), runID) - return err -} diff --git a/backtest/types.go b/backtest/types.go deleted file mode 100644 index f9c1295c..00000000 --- a/backtest/types.go +++ /dev/null @@ -1,179 +0,0 @@ -package backtest - -import "time" - -// RunState represents the current state of a backtest run. -type RunState string - -const ( - RunStateCreated RunState = "created" - RunStateRunning RunState = "running" - RunStatePaused RunState = "paused" - RunStateStopped RunState = "stopped" - RunStateCompleted RunState = "completed" - RunStateFailed RunState = "failed" - RunStateLiquidated RunState = "liquidated" -) - -// PositionSnapshot represents core position data for backtest state and persistence. -type PositionSnapshot struct { - Symbol string `json:"symbol"` - Side string `json:"side"` - Quantity float64 `json:"quantity"` - AvgPrice float64 `json:"avg_price"` - Leverage int `json:"leverage"` - LiquidationPrice float64 `json:"liquidation_price"` - MarginUsed float64 `json:"margin_used"` - OpenTime int64 `json:"open_time"` - AccumulatedFee float64 `json:"accumulated_fee,omitempty"` // Opening fees accumulated -} - -// BacktestState represents the real-time state during execution (in-memory state). -type BacktestState struct { - BarIndex int - BarTimestamp int64 - DecisionCycle int - - Cash float64 - Equity float64 - UnrealizedPnL float64 - RealizedPnL float64 - MaxEquity float64 - MinEquity float64 - MaxDrawdownPct float64 - Positions map[string]PositionSnapshot - LastUpdate time.Time - Liquidated bool - LiquidationNote string -} - -// EquityPoint represents a single point on the equity curve. -type EquityPoint struct { - Timestamp int64 `json:"ts"` - Equity float64 `json:"equity"` - Available float64 `json:"available"` - PnL float64 `json:"pnl"` - PnLPct float64 `json:"pnl_pct"` - DrawdownPct float64 `json:"dd_pct"` - Cycle int `json:"cycle"` -} - -// TradeEvent records a trade execution result or special event (such as liquidation). -type TradeEvent struct { - Timestamp int64 `json:"ts"` - Symbol string `json:"symbol"` - Action string `json:"action"` - Side string `json:"side,omitempty"` - Quantity float64 `json:"qty"` - Price float64 `json:"price"` - Fee float64 `json:"fee"` - Slippage float64 `json:"slippage"` - OrderValue float64 `json:"order_value"` - RealizedPnL float64 `json:"realized_pnl"` - Leverage int `json:"leverage,omitempty"` - Cycle int `json:"cycle"` - PositionAfter float64 `json:"position_after"` - LiquidationFlag bool `json:"liquidation"` - Note string `json:"note,omitempty"` -} - -// Metrics summarizes backtest performance metrics. -type Metrics struct { - TotalReturnPct float64 `json:"total_return_pct"` - MaxDrawdownPct float64 `json:"max_drawdown_pct"` - SharpeRatio float64 `json:"sharpe_ratio"` - ProfitFactor float64 `json:"profit_factor"` - WinRate float64 `json:"win_rate"` - Trades int `json:"trades"` - AvgWin float64 `json:"avg_win"` - AvgLoss float64 `json:"avg_loss"` - BestSymbol string `json:"best_symbol"` - WorstSymbol string `json:"worst_symbol"` - SymbolStats map[string]SymbolMetrics `json:"symbol_stats"` - Liquidated bool `json:"liquidated"` -} - -// SymbolMetrics records performance for a single symbol. -type SymbolMetrics struct { - TotalTrades int `json:"total_trades"` - WinningTrades int `json:"winning_trades"` - LosingTrades int `json:"losing_trades"` - TotalPnL float64 `json:"total_pnl"` - AvgPnL float64 `json:"avg_pnl"` - WinRate float64 `json:"win_rate"` -} - -// Checkpoint represents checkpoint information saved to disk for pause, resume, and crash recovery. -type Checkpoint struct { - BarIndex int `json:"bar_index"` - BarTimestamp int64 `json:"bar_ts"` - Cash float64 `json:"cash"` - Equity float64 `json:"equity"` - MaxEquity float64 `json:"max_equity"` - MinEquity float64 `json:"min_equity"` - MaxDrawdownPct float64 `json:"max_drawdown_pct"` - UnrealizedPnL float64 `json:"unrealized_pnl"` - RealizedPnL float64 `json:"realized_pnl"` - Positions []PositionSnapshot `json:"positions"` - DecisionCycle int `json:"decision_cycle"` - IndicatorsState map[string]map[string]any `json:"indicators_state,omitempty"` - RNGSeed int64 `json:"rng_seed,omitempty"` - AICacheRef string `json:"ai_cache_ref,omitempty"` - Liquidated bool `json:"liquidated"` - LiquidationNote string `json:"liquidation_note,omitempty"` -} - -// RunMetadata records the summary required for run.json. -type RunMetadata struct { - RunID string `json:"run_id"` - Label string `json:"label,omitempty"` - UserID string `json:"user_id,omitempty"` - LastError string `json:"last_error,omitempty"` - Version int `json:"version"` - State RunState `json:"state"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Summary RunSummary `json:"summary"` -} - -// RunSummary represents the summary field in run.json. -type RunSummary struct { - SymbolCount int `json:"symbol_count"` - DecisionTF string `json:"decision_tf"` - ProcessedBars int `json:"processed_bars"` - ProgressPct float64 `json:"progress_pct"` - EquityLast float64 `json:"equity_last"` - MaxDrawdownPct float64 `json:"max_drawdown_pct"` - Liquidated bool `json:"liquidated"` - LiquidationNote string `json:"liquidation_note,omitempty"` -} - -// StatusPayload is used for /status API responses. -type StatusPayload struct { - RunID string `json:"run_id"` - State RunState `json:"state"` - ProgressPct float64 `json:"progress_pct"` - ProcessedBars int `json:"processed_bars"` - CurrentTime int64 `json:"current_time"` - DecisionCycle int `json:"decision_cycle"` - Equity float64 `json:"equity"` - UnrealizedPnL float64 `json:"unrealized_pnl"` - RealizedPnL float64 `json:"realized_pnl"` - Positions []PositionStatus `json:"positions,omitempty"` - Note string `json:"note,omitempty"` - LastError string `json:"last_error,omitempty"` - LastUpdatedIso string `json:"last_updated_iso"` -} - -// PositionStatus represents a position with unrealized P&L for status display. -type PositionStatus struct { - Symbol string `json:"symbol"` - Side string `json:"side"` - Quantity float64 `json:"quantity"` - EntryPrice float64 `json:"entry_price"` - MarkPrice float64 `json:"mark_price"` - Leverage int `json:"leverage"` - UnrealizedPnL float64 `json:"unrealized_pnl"` - UnrealizedPnLPct float64 `json:"unrealized_pnl_pct"` - MarginUsed float64 `json:"margin_used"` -} diff --git a/docs/architecture/BACKTEST_MODULE.md b/docs/architecture/BACKTEST_MODULE.md deleted file mode 100644 index 820c1e2b..00000000 --- a/docs/architecture/BACKTEST_MODULE.md +++ /dev/null @@ -1,624 +0,0 @@ -# NOFX Backtest Module - Technical Documentation - -**Language:** [English](BACKTEST_MODULE.md) | [中文](BACKTEST_MODULE.zh-CN.md) - -## Overview - -This document describes the complete technical implementation of the NOFX backtest module, including configuration, historical data loading, simulation engine, AI decision making, performance metrics calculation, and result storage. - ---- - -## Complete Backtest Flow - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Backtest Execution Flow │ -└─────────────────────────────────────────────────────────────────┘ - -1. API Request: /backtest/start - ↓ -2. Manager.Start() - ├─ Validate config - ├─ Parse AI model - ├─ Create Runner instance - └─ Start runner.Start() (goroutine) - ↓ -3. Runner.Start() → Runner.loop() - └─ Iterate each decision time point: - ├─ DataFeed.BuildMarketData() [Build market data] - ├─ Check decision trigger [Every N bars] - ├─ buildDecisionContext() [Build decision context] - ├─ invokeAIWithRetry() [Call AI + cache] - ├─ executeDecision() [Execute trades] - ├─ checkLiquidation() [Check liquidation] - ├─ updateState() [Update state] - ├─ appendEquityPoint() [Record equity] - ├─ appendTradeEvent() [Record trades] - ├─ maybeCheckpoint() [Save checkpoint] - └─ persistMetrics() [Persist metrics] - ↓ -4. Complete/Failed - ├─ Calculate final metrics - ├─ Persist all results - └─ Release lock - ↓ -5. API Query: /backtest/metrics, /backtest/equity, /backtest/trades - └─ Load and return results -``` - ---- - -## 1. Configuration - -**Core File:** `backtest/config.go` - -### 1.1 Config Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `RunID` | string | (required) | Unique backtest run ID | -| `UserID` | string | "default" | User ID | -| `Symbols` | []string | (required) | Trading symbols list | -| `Timeframes` | []string | ["3m", "15m", "4h"] | K-line timeframes | -| `DecisionTimeframe` | string | Symbols[0] | Primary decision timeframe | -| `DecisionCadenceNBars` | int | 20 | Trigger decision every N bars | -| `StartTS`, `EndTS` | int64 | (required) | Backtest time range (Unix timestamp) | -| `InitialBalance` | float64 | 1000 | Initial balance (USD) | -| `FeeBps` | float64 | 5 | Trading fee (basis points) | -| `SlippageBps` | float64 | 2 | Slippage (basis points) | -| `FillPolicy` | string | "next_open" | Fill policy | -| `PromptVariant` | string | "baseline" | AI prompt variant | -| `CacheAI` | bool | false | Cache AI decisions | -| `Leverage` | LeverageConfig | BTC/ETH:5, Altcoin:5 | Leverage settings | - -### 1.2 Fill Policy - -```go -// backtest/config.go:163-179 -switch fillPolicy { -case "next_open": // Next bar open price -case "bar_vwap": // Current bar VWAP -case "mid": // Current bar (High+Low)/2 -default: // Mark Price -} -``` - -### 1.3 Config Example - -```go -cfg := backtest.BacktestConfig{ - RunID: "bt_20231215_150405", - Symbols: []string{"BTCUSDT", "ETHUSDT"}, - Timeframes: []string{"3m", "15m", "4h"}, - DecisionTimeframe: "3m", - DecisionCadenceNBars: 20, - StartTS: 1702566000, - EndTS: 1702652400, - InitialBalance: 10000, - FeeBps: 5, - SlippageBps: 2, - FillPolicy: "next_open", -} -``` - ---- - -## 2. Data Loading - -**Core File:** `backtest/datafeed.go` - -### 2.1 Data Loading Flow - -``` -1. NewDataFeed() - Initialize - ↓ -2. loadAll() - Load all historical data - ├─ Calculate buffer (200 bars before StartTS) - ├─ Call market.GetKlinesRange() to fetch data - ├─ Store in symbolSeries map - └─ Build decision timeline from primary timeframe - ↓ -3. BuildMarketData() - Build market data snapshot - ├─ Slice K-line data to current timestamp - ├─ Calculate technical indicators (EMA, MACD, RSI, ATR) - └─ Return market.Data structure -``` - -### 2.2 Data Structure - -```go -// DataFeed core structure -type DataFeed struct { - decisionTimes []int64 // Decision time points list - symbolSeries map[string]*symbolSeries // Data stored by symbol -} - -// Single symbol time series -type symbolSeries struct { - timeframes map[string]*timeframeSeries // Stored by timeframe -} - -// Single timeframe data -type timeframeSeries struct { - klines []market.Kline // K-line data - closeTimes []int64 // Close time index -} -``` - -### 2.3 Key Code References - -- Data fetching: `backtest/datafeed.go:48-93` -- Timeline generation: `backtest/datafeed.go:96-115` -- Market data assembly: `backtest/datafeed.go:141-171` - ---- - -## 3. Simulation Engine - -**Core File:** `backtest/runner.go` - -### 3.1 Main Loop - -```go -// backtest/runner.go:232-264 -func (r *Runner) loop() { - for _, ts := range r.feed.DecisionTimes() { - if r.isPaused() { - break - } - r.stepOnce(ts) - } -} -``` - -### 3.2 Single Step Execution - -```go -// backtest/runner.go:266-471 -func (r *Runner) stepOnce(ts int64) { - // 1. Get current bar timestamp - // 2. Build market data - // 3. Check decision trigger (every N bars) - // 4. Execute decision cycle (if triggered) - // 5. Check liquidation - // 6. Update state and record -} -``` - -### 3.3 State Management - -```go -// backtest/types.go:31-47 -type BacktestState struct { - BarIndex int // Current bar index - Cash float64 // Available balance - Equity float64 // Total equity - UnrealizedPnL float64 // Unrealized PnL - RealizedPnL float64 // Realized PnL - MaxEquity float64 // Peak equity - MinEquity float64 // Trough equity - MaxDrawdownPct float64 // Max drawdown - Positions map[string]*position // Positions -} -``` - ---- - -## 4. AI Decision Making - -**Core File:** `backtest/runner.go` - -### 4.1 Decision Context Building - -```go -// backtest/runner.go:473-532 -func (r *Runner) buildDecisionContext() *decision.Context { - return &decision.Context{ - CurrentTime: "2023-12-15 10:30:00 UTC", - RuntimeMinutes: elapsed, - CallCount: cycleNumber, - Account: { - TotalEquity, AvailableBalance, TotalPnL, MarginUsedPct - }, - Positions: []PositionInfo{...}, - CandidateCoins: []string{symbols...}, - MarketDataMap: map[symbol]*market.Data{...}, - MultiTFMarket: map[symbol]map[timeframe]*market.Data{...}, - } -} -``` - -### 4.2 AI Invocation - -```go -// backtest/runner.go:544-563 -func (r *Runner) invokeAIWithRetry() (*decision.FullDecision, error) { - // Max 3 retries - // Exponential backoff: 500ms, 1000ms, 1500ms - // Uses decision.GetFullDecisionWithStrategy() for unified prompt generation -} -``` - -### 4.3 AI Cache - -```go -// backtest/aicache.go:127-168 -// Cache key: SHA256(context payload) -// Contains: variant, timestamp, account, positions, market data -``` - -### 4.4 Supported AI Models - -| Model | Client File | -|-------|-------------| -| DeepSeek | `mcp/deepseek_client.go` | -| Qwen | `mcp/qwen_client.go` | -| Claude | `mcp/claude_client.go` | -| Gemini | `mcp/gemini_client.go` | -| Grok | `mcp/grok_client.go` | -| OpenAI | `mcp/openai_client.go` | -| Kimi | `mcp/kimi_client.go` | - ---- - -## 5. Performance Metrics - -**Core File:** `backtest/metrics.go` - -### 5.1 Metrics Calculation - -| Metric | Formula | Code Location | -|--------|---------|---------------| -| **Total Return** | (Final Equity - Initial) / Initial × 100 | metrics.go:36-42 | -| **Max Drawdown** | max((Peak - Current) / Peak × 100) | metrics.go:64-91 | -| **Sharpe Ratio** | Avg Return / Return StdDev | metrics.go:94-138 | -| **Win Rate** | Winning Trades / Total Trades × 100 | metrics.go:180-181 | -| **Profit Factor** | Total Profit / Total Loss | metrics.go:189-193 | - -### 5.2 Trade Statistics - -```go -// backtest/metrics.go:141-225 -type TradeMetrics struct { - TotalTrades int - WinningTrades int - LosingTrades int - AvgWin float64 - AvgLoss float64 - BestSymbol string - WorstSymbol string - SymbolStats map[string]*SymbolStat -} -``` - ---- - -## 6. Equity Curve - -**Core File:** `backtest/equity.go` - -### 6.1 Equity Point Structure - -```json -{ - "ts": 1702566000000, - "equity": 10500.50, - "available": 8000.00, - "pnl": 500.50, - "pnl_pct": 5.005, - "dd_pct": 2.34, - "cycle": 42 -} -``` - -### 6.2 Equity Update - -```go -// backtest/runner.go:829-872 -func (r *Runner) updateState() { - // 1. Calculate total equity: cash + margin + unrealized PnL - // 2. Track peak (MaxEquity) - // 3. Track trough (MinEquity) - // 4. Recalculate drawdown: (MaxEquity - Equity) / MaxEquity × 100 -} -``` - -### 6.3 Data Resampling - -```go -// backtest/equity.go:10-50 -func ResampleEquity(points []EquityPoint, timeframe string) []EquityPoint { - // Bucket by timeframe - // Keep last point in each bucket -} -``` - ---- - -## 7. Result Storage - -**Core Files:** `backtest/storage.go`, `store/backtest.go` - -### 7.1 File Storage Structure - -``` -backtests/ -├── / -│ ├── run.json # Run metadata -│ ├── checkpoint.json # Checkpoint (for resume) -│ ├── equity.jsonl # Equity curve (line-delimited JSON) -│ ├── trades.jsonl # Trade records (line-delimited JSON) -│ ├── metrics.json # Performance metrics -│ ├── progress.json # Progress info -│ ├── ai_cache.json # AI decision cache -│ └── decision_logs/ # Decision logs -│ ├── 0.json -│ ├── 1.json -│ └── ... -``` - -### 7.2 Database Schema - -```sql --- Backtest run metadata -CREATE TABLE backtest_runs ( - run_id TEXT PRIMARY KEY, - user_id TEXT, - config_json TEXT, - state TEXT, -- pending, running, completed, failed - processed_bars INTEGER, - progress_pct REAL, - equity_last REAL, - max_drawdown_pct REAL, - liquidated BOOLEAN, - ai_provider TEXT, - ai_model TEXT, - created_at DATETIME, - updated_at DATETIME -); - --- Equity curve -CREATE TABLE backtest_equity ( - id INTEGER PRIMARY KEY, - run_id TEXT, - ts INTEGER, - equity REAL, - available REAL, - pnl REAL, - pnl_pct REAL, - dd_pct REAL, - cycle INTEGER -); - --- Trade records -CREATE TABLE backtest_trades ( - id INTEGER PRIMARY KEY, - run_id TEXT, - ts INTEGER, - symbol TEXT, - action TEXT, - side TEXT, - qty REAL, - price REAL, - fee REAL, - slippage REAL, - realized_pnl REAL, - leverage INTEGER, - liquidation BOOLEAN -); - --- Performance metrics -CREATE TABLE backtest_metrics ( - run_id TEXT PRIMARY KEY, - payload BLOB, - updated_at DATETIME -); - --- Checkpoints (pause/resume) -CREATE TABLE backtest_checkpoints ( - run_id TEXT PRIMARY KEY, - payload BLOB, - updated_at DATETIME -); -``` - ---- - -## 8. API Endpoints - -**Core File:** `api/backtest.go` - -### 8.1 Endpoint List - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/backtest/start` | POST | Start backtest | -| `/backtest/pause` | POST | Pause backtest | -| `/backtest/resume` | POST | Resume backtest | -| `/backtest/stop` | POST | Stop backtest | -| `/backtest/status` | GET | Get status | -| `/backtest/runs` | GET | List all backtests | -| `/backtest/equity` | GET | Get equity curve | -| `/backtest/trades` | GET | Get trade records | -| `/backtest/metrics` | GET | Get performance metrics | -| `/backtest/trace` | GET | Get decision logs | -| `/backtest/export` | GET | Export ZIP | -| `/backtest/delete` | POST | Delete backtest | - -### 8.2 Request Examples - -```bash -# Start backtest -POST /backtest/start -{ - "config": { - "run_id": "bt_20231215", - "symbols": ["BTCUSDT", "ETHUSDT"], - "timeframes": ["3m", "15m", "4h"], - "start_ts": 1702566000, - "end_ts": 1702652400, - "initial_balance": 10000, - "ai_model_id": "model_001" - } -} - -# Get equity curve -GET /backtest/equity?run_id=bt_20231215&tf=1h&limit=1000 - -# Get metrics -GET /backtest/metrics?run_id=bt_20231215 -``` - -### 8.3 Response Examples - -```json -// Status response -{ - "run_id": "bt_20231215", - "state": "running", - "progress_pct": 45.5, - "processed_bars": 1234, - "equity": 10234.50, - "unrealized_pnl": 234.50 -} - -// Metrics response -{ - "total_return_pct": 12.34, - "max_drawdown_pct": 5.67, - "sharpe_ratio": 1.89, - "profit_factor": 2.34, - "win_rate": 65.5, - "trades": 123 -} -``` - ---- - -## 9. Account & Position Management - -**Core File:** `backtest/account.go` - -### 9.1 Position Structure - -```go -type position struct { - Symbol string - Side string // "long" or "short" - Quantity float64 - EntryPrice float64 - Leverage int - Margin float64 // Margin - Notional float64 // Notional value - LiquidationPrice float64 // Liquidation price - OpenTime int64 -} -``` - -### 9.2 Open Position Logic - -```go -// backtest/account.go:61-104 -func (a *BacktestAccount) Open(symbol, side string, qty, price float64, leverage int) { - // 1. Apply slippage - // 2. Calculate notional value (qty × price) - // 3. Calculate margin (notional / leverage) - // 4. Deduct margin + fees - // 5. Create/add to position - // 6. Calculate liquidation price -} -``` - -### 9.3 Close Position Logic - -```go -// backtest/account.go:106-140 -func (a *BacktestAccount) Close(symbol, side string, qty, price float64) { - // 1. Verify position exists - // 2. Apply slippage (reverse direction) - // 3. Calculate realized PnL - // long: (exit - entry) × qty - // short: (entry - exit) × qty - // 4. Return margin + PnL - fees - // 5. Update/delete position -} -``` - -### 9.4 Liquidation Price Calculation - -```go -// backtest/account.go:177-186 -func computeLiquidation(entry float64, leverage int, side string) float64 { - if side == "long" { - return entry * (1 - 1.0/float64(leverage)) // Long: liquidate on drop - } - return entry * (1 + 1.0/float64(leverage)) // Short: liquidate on rise -} -``` - ---- - -## 10. Checkpoint & Resume - -**Core File:** `backtest/runner.go` - -### 10.1 Checkpoint Structure - -```json -{ - "bar_index": 1234, - "bar_ts": 1702609200000, - "cash": 8000.00, - "equity": 10234.50, - "max_equity": 10500.00, - "max_drawdown_pct": 5.67, - "positions": [...], - "decision_cycle": 62, - "liquidated": false -} -``` - -### 10.2 Checkpoint Trigger - -```go -// backtest/runner.go:874-898 -func (r *Runner) maybeCheckpoint() { - // Save every N bars - // Or save every N seconds -} -``` - -### 10.3 Resume Flow - -```go -func (r *Runner) RestoreFromCheckpoint() { - // 1. Load checkpoint - // 2. Restore account state - // 3. Restore bar index (continue from next bar) - // 4. Restore equity curve, trade records -} -``` - ---- - -## Core File Index - -| Module | File | Key Methods | -|--------|------|-------------| -| **Config** | `backtest/config.go` | `BacktestConfig`, `Validate()` | -| **Data Loading** | `backtest/datafeed.go` | `NewDataFeed()`, `loadAll()`, `BuildMarketData()` | -| **Sim Engine** | `backtest/runner.go` | `Start()`, `loop()`, `stepOnce()` | -| **Decision** | `backtest/runner.go` | `buildDecisionContext()`, `invokeAIWithRetry()` | -| **Execution** | `backtest/runner.go` | `executeDecision()` | -| **Account** | `backtest/account.go` | `Open()`, `Close()`, `TotalEquity()` | -| **Metrics** | `backtest/metrics.go` | `CalculateMetrics()` | -| **Equity** | `backtest/equity.go` | `ResampleEquity()`, `LimitEquityPoints()` | -| **Storage** | `backtest/storage.go` | `SaveCheckpoint()`, `appendEquityPoint()` | -| **Database** | `store/backtest.go` | Schema and CRUD operations | -| **API** | `api/backtest.go` | HTTP handlers | -| **AI Cache** | `backtest/aicache.go` | `Get()`, `Put()`, `save()` | - ---- - -**Document Version:** 1.0.0 -**Last Updated:** 2025-01-15 diff --git a/docs/architecture/BACKTEST_MODULE.zh-CN.md b/docs/architecture/BACKTEST_MODULE.zh-CN.md deleted file mode 100644 index 8ec1938c..00000000 --- a/docs/architecture/BACKTEST_MODULE.zh-CN.md +++ /dev/null @@ -1,624 +0,0 @@ -# NOFX 回测模块技术文档 - -**语言:** [English](BACKTEST_MODULE.md) | [中文](BACKTEST_MODULE.zh-CN.md) - -## 概述 - -本文档详细描述 NOFX 回测模块的完整技术实现，包括配置、历史数据加载、模拟引擎、AI 决策、性能指标计算和结果存储。 - ---- - -## 完整回测流程图 - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ 回测执行流程 │ -└─────────────────────────────────────────────────────────────────┘ - -1. API 请求: /backtest/start - ↓ -2. Manager.Start() - ├─ 验证配置 - ├─ 解析 AI 模型 - ├─ 创建 Runner 实例 - └─ 启动 runner.Start() (goroutine) - ↓ -3. Runner.Start() → Runner.loop() - └─ 遍历每个决策时间点: - ├─ DataFeed.BuildMarketData() [构建市场数据] - ├─ 检查决策触发条件 [每 N 根 K 线] - ├─ buildDecisionContext() [构建决策上下文] - ├─ invokeAIWithRetry() [调用 AI + 缓存] - ├─ executeDecision() [执行交易] - ├─ checkLiquidation() [检查爆仓] - ├─ updateState() [更新状态] - ├─ appendEquityPoint() [记录权益] - ├─ appendTradeEvent() [记录交易] - ├─ maybeCheckpoint() [保存检查点] - └─ persistMetrics() [持久化指标] - ↓ -4. 完成/失败 - ├─ 计算最终指标 - ├─ 持久化所有结果 - └─ 释放锁 - ↓ -5. API 查询: /backtest/metrics, /backtest/equity, /backtest/trades - └─ 加载并返回结果 -``` - ---- - -## 1. 回测配置 (Configuration) - -**核心文件:** `backtest/config.go` - -### 1.1 配置参数 - -| 参数 | 类型 | 默认值 | 说明 | -|------|------|--------|------| -| `RunID` | string | (必填) | 回测运行唯一标识 | -| `UserID` | string | "default" | 用户 ID | -| `Symbols` | []string | (必填) | 交易币种列表 | -| `Timeframes` | []string | ["3m", "15m", "4h"] | K 线周期 | -| `DecisionTimeframe` | string | Symbols[0] | 主决策周期 | -| `DecisionCadenceNBars` | int | 20 | 每 N 根 K 线触发一次决策 | -| `StartTS`, `EndTS` | int64 | (必填) | 回测时间范围 (Unix 时间戳) | -| `InitialBalance` | float64 | 1000 | 初始资金 (USD) | -| `FeeBps` | float64 | 5 | 手续费 (基点) | -| `SlippageBps` | float64 | 2 | 滑点 (基点) | -| `FillPolicy` | string | "next_open" | 成交策略 | -| `PromptVariant` | string | "baseline" | AI 提示词变体 | -| `CacheAI` | bool | false | 是否缓存 AI 决策 | -| `Leverage` | LeverageConfig | BTC/ETH:5, Altcoin:5 | 杠杆设置 | - -### 1.2 成交策略 (Fill Policy) - -```go -// backtest/config.go:163-179 -switch fillPolicy { -case "next_open": // 下一根 K 线开盘价 -case "bar_vwap": // 当前 K 线 VWAP -case "mid": // 当前 K 线 (High+Low)/2 -default: // Mark Price -} -``` - -### 1.3 配置示例 - -```go -cfg := backtest.BacktestConfig{ - RunID: "bt_20231215_150405", - Symbols: []string{"BTCUSDT", "ETHUSDT"}, - Timeframes: []string{"3m", "15m", "4h"}, - DecisionTimeframe: "3m", - DecisionCadenceNBars: 20, - StartTS: 1702566000, - EndTS: 1702652400, - InitialBalance: 10000, - FeeBps: 5, - SlippageBps: 2, - FillPolicy: "next_open", -} -``` - ---- - -## 2. 历史数据加载 (Data Loading) - -**核心文件:** `backtest/datafeed.go` - -### 2.1 数据加载流程 - -``` -1. NewDataFeed() - 初始化 - ↓ -2. loadAll() - 加载所有历史数据 - ├─ 计算缓冲区 (StartTS 前 200 根 K 线) - ├─ 调用 market.GetKlinesRange() 获取数据 - ├─ 存储到 symbolSeries map - └─ 从主周期构建决策时间线 - ↓ -3. BuildMarketData() - 构建市场数据快照 - ├─ 切片 K 线数据到当前时间戳 - ├─ 计算技术指标 (EMA, MACD, RSI, ATR) - └─ 返回 market.Data 结构 -``` - -### 2.2 数据结构 - -```go -// DataFeed 核心结构 -type DataFeed struct { - decisionTimes []int64 // 决策时间点列表 - symbolSeries map[string]*symbolSeries // 按币种存储的数据 -} - -// 单币种时间序列 -type symbolSeries struct { - timeframes map[string]*timeframeSeries // 按周期存储 -} - -// 单周期数据 -type timeframeSeries struct { - klines []market.Kline // K 线数据 - closeTimes []int64 // 收盘时间索引 -} -``` - -### 2.3 关键代码引用 - -- 数据获取: `backtest/datafeed.go:48-93` -- 时间线生成: `backtest/datafeed.go:96-115` -- 市场数据组装: `backtest/datafeed.go:141-171` - ---- - -## 3. 模拟引擎 (Simulation Engine) - -**核心文件:** `backtest/runner.go` - -### 3.1 主循环 - -```go -// backtest/runner.go:232-264 -func (r *Runner) loop() { - for _, ts := range r.feed.DecisionTimes() { - if r.isPaused() { - break - } - r.stepOnce(ts) - } -} -``` - -### 3.2 单步执行 - -```go -// backtest/runner.go:266-471 -func (r *Runner) stepOnce(ts int64) { - // 1. 获取当前 K 线时间戳 - // 2. 构建市场数据 - // 3. 检查决策触发条件 (每 N 根 K 线) - // 4. 执行决策周期 (如果触发) - // 5. 检查爆仓 - // 6. 更新状态并记录 -} -``` - -### 3.3 状态管理 - -```go -// backtest/types.go:31-47 -type BacktestState struct { - BarIndex int // 当前 K 线索引 - Cash float64 // 可用余额 - Equity float64 // 总权益 - UnrealizedPnL float64 // 未实现盈亏 - RealizedPnL float64 // 已实现盈亏 - MaxEquity float64 // 最高权益 - MinEquity float64 // 最低权益 - MaxDrawdownPct float64 // 最大回撤 - Positions map[string]*position // 持仓 -} -``` - ---- - -## 4. AI 决策 (AI Decision Making) - -**核心文件:** `backtest/runner.go` - -### 4.1 决策上下文构建 - -```go -// backtest/runner.go:473-532 -func (r *Runner) buildDecisionContext() *decision.Context { - return &decision.Context{ - CurrentTime: "2023-12-15 10:30:00 UTC", - RuntimeMinutes: elapsed, - CallCount: cycleNumber, - Account: { - TotalEquity, AvailableBalance, TotalPnL, MarginUsedPct - }, - Positions: []PositionInfo{...}, - CandidateCoins: []string{symbols...}, - MarketDataMap: map[symbol]*market.Data{...}, - MultiTFMarket: map[symbol]map[timeframe]*market.Data{...}, - } -} -``` - -### 4.2 AI 调用 - -```go -// backtest/runner.go:544-563 -func (r *Runner) invokeAIWithRetry() (*decision.FullDecision, error) { - // 最多重试 3 次 - // 指数退避: 500ms, 1000ms, 1500ms - // 使用 decision.GetFullDecisionWithStrategy() 统一提示词生成 -} -``` - -### 4.3 AI 缓存 - -```go -// backtest/aicache.go:127-168 -// 缓存键: SHA256(context payload) -// 包含: variant, timestamp, account, positions, market data -``` - -### 4.4 支持的 AI 模型 - -| 模型 | 客户端文件 | -|------|-----------| -| DeepSeek | `mcp/deepseek_client.go` | -| Qwen | `mcp/qwen_client.go` | -| Claude | `mcp/claude_client.go` | -| Gemini | `mcp/gemini_client.go` | -| Grok | `mcp/grok_client.go` | -| OpenAI | `mcp/openai_client.go` | -| Kimi | `mcp/kimi_client.go` | - ---- - -## 5. 性能指标 (Performance Metrics) - -**核心文件:** `backtest/metrics.go` - -### 5.1 指标计算 - -| 指标 | 公式 | 代码位置 | -|------|------|----------| -| **总收益率** | (最终权益 - 初始资金) / 初始资金 × 100 | metrics.go:36-42 | -| **最大回撤** | max((峰值 - 当前) / 峰值 × 100) | metrics.go:64-91 | -| **夏普比率** | 平均收益 / 收益标准差 | metrics.go:94-138 | -| **胜率** | 盈利交易数 / 总交易数 × 100 | metrics.go:180-181 | -| **盈亏比** | 总盈利 / 总亏损 | metrics.go:189-193 | - -### 5.2 交易统计 - -```go -// backtest/metrics.go:141-225 -type TradeMetrics struct { - TotalTrades int - WinningTrades int - LosingTrades int - AvgWin float64 - AvgLoss float64 - BestSymbol string - WorstSymbol string - SymbolStats map[string]*SymbolStat -} -``` - ---- - -## 6. 权益曲线 (Equity Curve) - -**核心文件:** `backtest/equity.go` - -### 6.1 权益点结构 - -```json -{ - "ts": 1702566000000, - "equity": 10500.50, - "available": 8000.00, - "pnl": 500.50, - "pnl_pct": 5.005, - "dd_pct": 2.34, - "cycle": 42 -} -``` - -### 6.2 权益更新 - -```go -// backtest/runner.go:829-872 -func (r *Runner) updateState() { - // 1. 计算总权益: cash + margin + 未实现盈亏 - // 2. 追踪峰值 (MaxEquity) - // 3. 追踪谷值 (MinEquity) - // 4. 重新计算回撤: (MaxEquity - Equity) / MaxEquity × 100 -} -``` - -### 6.3 数据重采样 - -```go -// backtest/equity.go:10-50 -func ResampleEquity(points []EquityPoint, timeframe string) []EquityPoint { - // 按时间周期分桶 - // 保留每个桶的最后一个点 -} -``` - ---- - -## 7. 结果存储 (Result Storage) - -**核心文件:** `backtest/storage.go`, `store/backtest.go` - -### 7.1 文件存储结构 - -``` -backtests/ -├── / -│ ├── run.json # 运行元数据 -│ ├── checkpoint.json # 检查点 (用于恢复) -│ ├── equity.jsonl # 权益曲线 (逐行 JSON) -│ ├── trades.jsonl # 交易记录 (逐行 JSON) -│ ├── metrics.json # 性能指标 -│ ├── progress.json # 进度信息 -│ ├── ai_cache.json # AI 决策缓存 -│ └── decision_logs/ # 决策日志 -│ ├── 0.json -│ ├── 1.json -│ └── ... -``` - -### 7.2 数据库表结构 - -```sql --- 回测运行元数据 -CREATE TABLE backtest_runs ( - run_id TEXT PRIMARY KEY, - user_id TEXT, - config_json TEXT, - state TEXT, -- pending, running, completed, failed - processed_bars INTEGER, - progress_pct REAL, - equity_last REAL, - max_drawdown_pct REAL, - liquidated BOOLEAN, - ai_provider TEXT, - ai_model TEXT, - created_at DATETIME, - updated_at DATETIME -); - --- 权益曲线 -CREATE TABLE backtest_equity ( - id INTEGER PRIMARY KEY, - run_id TEXT, - ts INTEGER, - equity REAL, - available REAL, - pnl REAL, - pnl_pct REAL, - dd_pct REAL, - cycle INTEGER -); - --- 交易记录 -CREATE TABLE backtest_trades ( - id INTEGER PRIMARY KEY, - run_id TEXT, - ts INTEGER, - symbol TEXT, - action TEXT, - side TEXT, - qty REAL, - price REAL, - fee REAL, - slippage REAL, - realized_pnl REAL, - leverage INTEGER, - liquidation BOOLEAN -); - --- 性能指标 -CREATE TABLE backtest_metrics ( - run_id TEXT PRIMARY KEY, - payload BLOB, - updated_at DATETIME -); - --- 检查点 (暂停/恢复) -CREATE TABLE backtest_checkpoints ( - run_id TEXT PRIMARY KEY, - payload BLOB, - updated_at DATETIME -); -``` - ---- - -## 8. API 接口 - -**核心文件:** `api/backtest.go` - -### 8.1 接口列表 - -| 接口 | 方法 | 说明 | -|------|------|------| -| `/backtest/start` | POST | 开始回测 | -| `/backtest/pause` | POST | 暂停回测 | -| `/backtest/resume` | POST | 恢复回测 | -| `/backtest/stop` | POST | 停止回测 | -| `/backtest/status` | GET | 获取状态 | -| `/backtest/runs` | GET | 列出所有回测 | -| `/backtest/equity` | GET | 获取权益曲线 | -| `/backtest/trades` | GET | 获取交易记录 | -| `/backtest/metrics` | GET | 获取性能指标 | -| `/backtest/trace` | GET | 获取决策日志 | -| `/backtest/export` | GET | 导出 ZIP | -| `/backtest/delete` | POST | 删除回测 | - -### 8.2 请求示例 - -```bash -# 开始回测 -POST /backtest/start -{ - "config": { - "run_id": "bt_20231215", - "symbols": ["BTCUSDT", "ETHUSDT"], - "timeframes": ["3m", "15m", "4h"], - "start_ts": 1702566000, - "end_ts": 1702652400, - "initial_balance": 10000, - "ai_model_id": "model_001" - } -} - -# 获取权益曲线 -GET /backtest/equity?run_id=bt_20231215&tf=1h&limit=1000 - -# 获取指标 -GET /backtest/metrics?run_id=bt_20231215 -``` - -### 8.3 响应示例 - -```json -// 状态响应 -{ - "run_id": "bt_20231215", - "state": "running", - "progress_pct": 45.5, - "processed_bars": 1234, - "equity": 10234.50, - "unrealized_pnl": 234.50 -} - -// 指标响应 -{ - "total_return_pct": 12.34, - "max_drawdown_pct": 5.67, - "sharpe_ratio": 1.89, - "profit_factor": 2.34, - "win_rate": 65.5, - "trades": 123 -} -``` - ---- - -## 9. 账户与持仓管理 - -**核心文件:** `backtest/account.go` - -### 9.1 持仓结构 - -```go -type position struct { - Symbol string - Side string // "long" 或 "short" - Quantity float64 - EntryPrice float64 - Leverage int - Margin float64 // 保证金 - Notional float64 // 名义价值 - LiquidationPrice float64 // 爆仓价格 - OpenTime int64 -} -``` - -### 9.2 开仓逻辑 - -```go -// backtest/account.go:61-104 -func (a *BacktestAccount) Open(symbol, side string, qty, price float64, leverage int) { - // 1. 应用滑点 - // 2. 计算名义价值 (qty × price) - // 3. 计算保证金 (notional / leverage) - // 4. 扣除保证金 + 手续费 - // 5. 创建/加仓 - // 6. 计算爆仓价格 -} -``` - -### 9.3 平仓逻辑 - -```go -// backtest/account.go:106-140 -func (a *BacktestAccount) Close(symbol, side string, qty, price float64) { - // 1. 验证持仓存在 - // 2. 应用滑点 (反向) - // 3. 计算已实现盈亏 - // long: (exit - entry) × qty - // short: (entry - exit) × qty - // 4. 返还保证金 + 盈亏 - 手续费 - // 5. 更新/删除持仓 -} -``` - -### 9.4 爆仓价格计算 - -```go -// backtest/account.go:177-186 -func computeLiquidation(entry float64, leverage int, side string) float64 { - if side == "long" { - return entry * (1 - 1.0/float64(leverage)) // 做多: 下跌爆仓 - } - return entry * (1 + 1.0/float64(leverage)) // 做空: 上涨爆仓 -} -``` - ---- - -## 10. 检查点与恢复 - -**核心文件:** `backtest/runner.go` - -### 10.1 检查点结构 - -```json -{ - "bar_index": 1234, - "bar_ts": 1702609200000, - "cash": 8000.00, - "equity": 10234.50, - "max_equity": 10500.00, - "max_drawdown_pct": 5.67, - "positions": [...], - "decision_cycle": 62, - "liquidated": false -} -``` - -### 10.2 检查点触发 - -```go -// backtest/runner.go:874-898 -func (r *Runner) maybeCheckpoint() { - // 每 N 根 K 线保存 - // 或每 N 秒保存 -} -``` - -### 10.3 恢复流程 - -```go -func (r *Runner) RestoreFromCheckpoint() { - // 1. 加载检查点 - // 2. 恢复账户状态 - // 3. 恢复 K 线索引 (从下一根继续) - // 4. 恢复权益曲线、交易记录 -} -``` - ---- - -## 核心文件索引 - -| 模块 | 文件 | 关键方法 | -|------|------|----------| -| **配置** | `backtest/config.go` | `BacktestConfig`, `Validate()` | -| **数据加载** | `backtest/datafeed.go` | `NewDataFeed()`, `loadAll()`, `BuildMarketData()` | -| **模拟引擎** | `backtest/runner.go` | `Start()`, `loop()`, `stepOnce()` | -| **决策** | `backtest/runner.go` | `buildDecisionContext()`, `invokeAIWithRetry()` | -| **执行** | `backtest/runner.go` | `executeDecision()` | -| **账户** | `backtest/account.go` | `Open()`, `Close()`, `TotalEquity()` | -| **指标** | `backtest/metrics.go` | `CalculateMetrics()` | -| **权益** | `backtest/equity.go` | `ResampleEquity()`, `LimitEquityPoints()` | -| **存储** | `backtest/storage.go` | `SaveCheckpoint()`, `appendEquityPoint()` | -| **数据库** | `store/backtest.go` | 表结构和 CRUD 操作 | -| **API** | `api/backtest.go` | HTTP 处理器 | -| **AI 缓存** | `backtest/aicache.go` | `Get()`, `Put()`, `save()` | - ---- - -**文档版本:** 1.0.0 -**最后更新:** 2025-01-15 diff --git a/docs/architecture/README.md b/docs/architecture/README.md index fd20dde3..3f18083c 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -24,12 +24,12 @@ NOFX is a full-stack AI trading platform for cryptocurrency and US stock markets │ NOFX Platform │ ├─────────────────────────────────────────────────────────────────────────────┤ │ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐│ -│ │ Strategy │ │ Backtest │ │ Live Trading ││ -│ │ Studio │ │ Engine │ │ (Auto Trader) ││ -│ └──────┬──────┘ └──────┬──────┘ └──────────┬──────────┘│ -│ │ │ │ │ -│ └────────────────┴────────────────────┘ │ +│ ┌─────────────┐ ┌─────────────────────────────────────┐│ +│ │ Strategy │ │ Live Trading ││ +│ │ Studio │ │ (Auto Trader) ││ +│ └──────┬──────┘ └──────────────────┬──────────────────┘│ +│ │ │ │ +│ └────────────────────────────┘ │ │ │ │ │ ┌─────────▼─────────┐ │ │ │ Core Services │ │ @@ -57,7 +57,6 @@ NOFX is a full-stack AI trading platform for cryptocurrency and US stock markets | Module | Description | Documentation | |--------|-------------|---------------| | **Strategy Studio** | Strategy configuration, coin selection, data assembly, AI prompts | [STRATEGY_MODULE.md](STRATEGY_MODULE.md) | -| **Backtest Engine** | Historical simulation, performance metrics, AI decision replay | [BACKTEST_MODULE.md](BACKTEST_MODULE.md) | ### Module Overview @@ -71,16 +70,6 @@ Complete strategy configuration system including: **[Read Full Documentation →](STRATEGY_MODULE.md)** -#### Backtest Module -Historical trading simulation engine: -- Multi-symbol, multi-timeframe backtesting -- AI decision replay with caching -- Performance metrics (Sharpe, drawdown, win rate) -- Real-time progress streaming via SSE -- Checkpoint and resume support - -**[Read Full Documentation →](BACKTEST_MODULE.md)** - --- ## Project Structure @@ -91,7 +80,6 @@ nofx/ ├── api/ # HTTP API (Gin framework) ├── trader/ # Trading execution layer ├── strategy/ # Strategy engine -├── backtest/ # Backtest simulation engine ├── market/ # Market data service ├── mcp/ # AI model clients ├── store/ # Database operations @@ -131,7 +119,6 @@ nofx/ ## Quick Links - [Strategy Module](STRATEGY_MODULE.md) - How strategies work -- [Backtest Module](BACKTEST_MODULE.md) - How backtesting works - [Getting Started](../getting-started/README.md) - Setup guide - [FAQ](../faq/README.md) - Frequently asked questions diff --git a/docs/architecture/README.zh-CN.md b/docs/architecture/README.zh-CN.md index f3a16b59..ae4d9228 100644 --- a/docs/architecture/README.zh-CN.md +++ b/docs/architecture/README.zh-CN.md @@ -24,12 +24,12 @@ NOFX 是一个支持加密货币和美股市场的全栈 AI 交易平台： │ NOFX 平台 │ ├─────────────────────────────────────────────────────────────────────────────┤ │ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐│ -│ │ 策略 │ │ 回测 │ │ 实盘交易 ││ -│ │ 工作室 │ │ 引擎 │ │ (自动交易员) ││ -│ └──────┬──────┘ └──────┬──────┘ └──────────┬──────────┘│ -│ │ │ │ │ -│ └────────────────┴────────────────────┘ │ +│ ┌─────────────┐ ┌─────────────────────────────────────┐│ +│ │ 策略 │ │ 实盘交易 ││ +│ │ 工作室 │ │ (自动交易员) ││ +│ └──────┬──────┘ └──────────────────┬──────────────────┘│ +│ │ │ │ +│ └────────────────────────────┘ │ │ │ │ │ ┌─────────▼─────────┐ │ │ │ 核心服务 │ │ @@ -57,7 +57,6 @@ NOFX 是一个支持加密货币和美股市场的全栈 AI 交易平台： | 模块 | 描述 | 文档 | |------|------|------| | **策略工作室** | 策略配置、币种选择、数据组装、AI 提示词 | [STRATEGY_MODULE.md](STRATEGY_MODULE.md) | -| **回测引擎** | 历史模拟、性能指标、AI 决策回放 | [BACKTEST_MODULE.md](BACKTEST_MODULE.md) | ### 模块概览 @@ -71,16 +70,6 @@ NOFX 是一个支持加密货币和美股市场的全栈 AI 交易平台： **[阅读完整文档 →](STRATEGY_MODULE.md)** -#### 回测模块 -历史交易模拟引擎： -- 多币种、多时间周期回测 -- AI 决策回放与缓存 -- 性能指标（夏普比率、最大回撤、胜率） -- SSE 实时进度推送 -- 断点续测支持 - -**[阅读完整文档 →](BACKTEST_MODULE.md)** - --- ## 项目结构 @@ -91,7 +80,6 @@ nofx/ ├── api/ # HTTP API (Gin 框架) ├── trader/ # 交易执行层 ├── strategy/ # 策略引擎 -├── backtest/ # 回测模拟引擎 ├── market/ # 行情数据服务 ├── mcp/ # AI 模型客户端 ├── store/ # 数据库操作 @@ -131,7 +119,6 @@ nofx/ ## 快速链接 - [策略模块](STRATEGY_MODULE.md) - 策略如何运作 -- [回测模块](BACKTEST_MODULE.md) - 回测如何运作 - [快速开始](../getting-started/README.zh-CN.md) - 部署指南 - [常见问题](../faq/README.md) - FAQ diff --git a/docs/i18n/vi/README.md b/docs/i18n/vi/README.md index 0eafcd48..e89805ef 100644 --- a/docs/i18n/vi/README.md +++ b/docs/i18n/vi/README.md @@ -79,7 +79,6 @@ Tương thích với **[ClawRouter](https://github.com/BlockRunAI/ClawRouter)** | **Strategy Studio** | Trình xây dựng trực quan — nguồn coin, chỉ báo, kiểm soát rủi ro | | **AI Competition** | AI cạnh tranh thời gian thực, bảng xếp hạng hiệu suất | | **Telegram Agent** | Chat với trợ lý giao dịch — streaming, gọi công cụ, bộ nhớ | -| **Backtest Lab** | Mô phỏng lịch sử, đường vốn và chỉ số hiệu suất | | **Dashboard** | Vị thế trực tiếp, P/L, nhật ký quyết định AI với Chain of Thought | ### Thị trường diff --git a/docs/i18n/zh-CN/README.md b/docs/i18n/zh-CN/README.md index d25edb6e..28bc8565 100644 --- a/docs/i18n/zh-CN/README.md +++ b/docs/i18n/zh-CN/README.md @@ -191,7 +191,6 @@ curl -fsSL https://raw.githubusercontent.com/NoFxAiOS/nofx/main/install.sh | bas |:--|:--| | [架构概览](../../architecture/README.md) | 系统设计和模块索引 | | [策略模块](../../architecture/STRATEGY_MODULE.md) | 币种选择、AI 提示词、执行 | -| [回测模块](../../architecture/BACKTEST_MODULE.md) | 历史模拟、指标计算 | | [常见问题](../../faq/README.md) | FAQ | | [快速开始](../../getting-started/README.md) | 部署指南 | diff --git a/docs/plans/2026-03-06-telegram-agent-redesign.md b/docs/plans/2026-03-06-telegram-agent-redesign.md index 9764d985..cde09a7c 100644 --- a/docs/plans/2026-03-06-telegram-agent-redesign.md +++ b/docs/plans/2026-03-06-telegram-agent-redesign.md @@ -243,7 +243,6 @@ s.route(protected, "GET", "/statistics", "Trading statistics (?trader_id Note: keep the existing special-case handlers that don't use `s.route` unchanged: - `api.Any("/health", ...)` — health check, no need to document - `api.GET("/crypto/...")` — crypto/encryption routes, bot doesn't need these -- `backtest.*` routes (registered separately) — add descriptions to the backtest group similarly **Step 3: Build** diff --git a/docs/plans/2026-03-06-telegram-bot.md b/docs/plans/2026-03-06-telegram-bot.md index 1627dce5..1365259c 100644 --- a/docs/plans/2026-03-06-telegram-bot.md +++ b/docs/plans/2026-03-06-telegram-bot.md @@ -993,7 +993,7 @@ func Start(cfg *config.Config, st *store.Store, tm *manager.TraderManager) { logger.Infof("🤖 Telegram bot started: @%s", bot.Self.UserName) - // Build the LLM client for intent parsing (use DeepSeek by default, same as backtest) + // Build the LLM client for intent parsing (use DeepSeek by default) llmClient := mcp.New() // Configure with whatever key is available in env (intent parsing is lightweight) // The service layer will use store to get user-configured models for actual trading diff --git a/docs/research/AI-Trader-Analysis-Report.md b/docs/research/AI-Trader-Analysis-Report.md index f0ad4838..b73c8329 100644 --- a/docs/research/AI-Trader-Analysis-Report.md +++ b/docs/research/AI-Trader-Analysis-Report.md @@ -2488,7 +2488,6 @@ KNOWN_ISSUES = [ 3. **金融量化** - Lo, Andrew W. "The Adaptive Markets Hypothesis." Journal of Portfolio Management, 2004. - - Bailey et al. "The Probability of Backtest Overfitting." Journal of Computational Finance, 2014. ### 13.3 代码示例索引 diff --git a/main.go b/main.go index 4dcf8bb6..7fa7295c 100644 --- a/main.go +++ b/main.go @@ -3,13 +3,11 @@ package main import ( "nofx/api" "nofx/auth" - "nofx/backtest" "nofx/config" "nofx/crypto" "nofx/telemetry" "nofx/logger" "nofx/manager" - "nofx/mcp" _ "nofx/mcp/payment" _ "nofx/mcp/provider" "nofx/store" @@ -81,7 +79,6 @@ func main() { logger.Fatalf("❌ Failed to initialize database: %v", err) } defer st.Close() - backtest.UseDatabaseWithType(st.DB(), st.DBType() == store.DBTypePostgres) // Initialize installation ID for experience improvement (anonymous statistics) initInstallationID(st) @@ -98,13 +95,8 @@ func main() { // time.Sleep(500 * time.Millisecond) logger.Info("📊 Using CoinAnk API for all market data (WebSocket cache disabled)") - // Create TraderManager and BacktestManager + // Create TraderManager traderManager := manager.NewTraderManager() - mcpClient := newSharedMCPClient() - backtestManager := backtest.NewManager(mcpClient) - if err := backtestManager.RestoreRuns(); err != nil { - logger.Warnf("⚠️ Failed to restore backtest history: %v", err) - } // Load all traders from database to memory (may auto-start traders with IsRunning=true) if err := traderManager.LoadTradersFromStore(st); err != nil { @@ -132,7 +124,7 @@ func main() { } // Start API server - server := api.NewServer(traderManager, st, cryptoService, backtestManager, cfg.APIServerPort) + server := api.NewServer(traderManager, st, cryptoService, cfg.APIServerPort) // Create hot-reload channel for Telegram bot; wire it to the API server // so that POST /api/telegram can trigger a bot restart when the token changes. @@ -163,16 +155,6 @@ func main() { logger.Info("✅ System shut down safely") } -// newSharedMCPClient creates a shared MCP AI client (for backtesting) -func newSharedMCPClient() mcp.AIClient { - apiKey := os.Getenv("DEEPSEEK_API_KEY") - if apiKey == "" { - logger.Warn("⚠️ DEEPSEEK_API_KEY not set, AI features will be unavailable") - return nil - } - return mcp.NewAIClientByProvider("deepseek") -} - // initInstallationID initializes the anonymous installation ID for experience improvement // This ID is persisted in database and used for anonymous usage statistics func initInstallationID(st *store.Store) { diff --git a/market/data.go b/market/data.go index 99dbfc9c..41548348 100644 --- a/market/data.go +++ b/market/data.go @@ -602,7 +602,7 @@ func parseFloat(v interface{}) (float64, error) { } } -// BuildDataFromKlines constructs market data snapshot from preloaded K-line series (for backtesting/simulation). +// BuildDataFromKlines constructs market data snapshot from preloaded K-line series. func BuildDataFromKlines(symbol string, primary []Kline, longer []Kline) (*Data, error) { if len(primary) == 0 { return nil, fmt.Errorf("primary series is empty") diff --git a/screenshots/backtest-lab.png b/screenshots/backtest-lab.png deleted file mode 100644 index 41f433d5..00000000 Binary files a/screenshots/backtest-lab.png and /dev/null differ diff --git a/store/backtest.go b/store/backtest.go deleted file mode 100644 index a77dfe16..00000000 --- a/store/backtest.go +++ /dev/null @@ -1,574 +0,0 @@ -package store - -import ( - "encoding/json" - "fmt" - "time" - - "gorm.io/gorm" -) - -// BacktestStore backtest data storage -type BacktestStore struct { - db *gorm.DB -} - -// NewBacktestStore creates a new backtest store -func NewBacktestStore(db *gorm.DB) *BacktestStore { - return &BacktestStore{db: db} -} - -// isPostgres checks if the database is PostgreSQL -func (s *BacktestStore) isPostgres() bool { - return s.db.Dialector.Name() == "postgres" -} - -// RunState backtest state -type RunState string - -const ( - RunStateCreated RunState = "created" - RunStateRunning RunState = "running" - RunStatePaused RunState = "paused" - RunStateCompleted RunState = "completed" - RunStateFailed RunState = "failed" -) - -// RunMetadata backtest metadata -type RunMetadata struct { - RunID string `json:"run_id"` - UserID string `json:"user_id"` - Version int `json:"version"` - State RunState `json:"state"` - Label string `json:"label"` - LastError string `json:"last_error"` - Summary RunSummary `json:"summary"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -// RunSummary backtest summary -type RunSummary struct { - SymbolCount int `json:"symbol_count"` - DecisionTF string `json:"decision_tf"` - ProcessedBars int `json:"processed_bars"` - ProgressPct float64 `json:"progress_pct"` - EquityLast float64 `json:"equity_last"` - MaxDrawdownPct float64 `json:"max_drawdown_pct"` - Liquidated bool `json:"liquidated"` - LiquidationNote string `json:"liquidation_note"` -} - -// EquityPoint equity point -type EquityPoint struct { - Timestamp int64 `json:"timestamp"` - Equity float64 `json:"equity"` - Available float64 `json:"available"` - PnL float64 `json:"pnl"` - PnLPct float64 `json:"pnl_pct"` - DrawdownPct float64 `json:"drawdown_pct"` - Cycle int `json:"cycle"` -} - -// TradeEvent trade event -type TradeEvent struct { - Timestamp int64 `json:"timestamp"` - Symbol string `json:"symbol"` - Action string `json:"action"` - Side string `json:"side"` - Quantity float64 `json:"quantity"` - Price float64 `json:"price"` - Fee float64 `json:"fee"` - Slippage float64 `json:"slippage"` - OrderValue float64 `json:"order_value"` - RealizedPnL float64 `json:"realized_pnl"` - Leverage int `json:"leverage"` - Cycle int `json:"cycle"` - PositionAfter float64 `json:"position_after"` - LiquidationFlag bool `json:"liquidation_flag"` - Note string `json:"note"` -} - -// RunIndexEntry backtest index entry -type RunIndexEntry struct { - RunID string `json:"run_id"` - State string `json:"state"` - Symbols []string `json:"symbols"` - DecisionTF string `json:"decision_tf"` - EquityLast float64 `json:"equity_last"` - MaxDrawdownPct float64 `json:"max_drawdown_pct"` - StartTS int64 `json:"start_ts"` - EndTS int64 `json:"end_ts"` - CreatedAtISO string `json:"created_at"` - UpdatedAtISO string `json:"updated_at"` -} - -// BacktestRun GORM model for backtest_runs table -type BacktestRun struct { - RunID string `gorm:"column:run_id;primaryKey"` - UserID string `gorm:"column:user_id;not null;default:''"` - ConfigJSON []byte `gorm:"column:config_json"` - State string `gorm:"column:state;not null;default:created"` - Label string `gorm:"column:label;default:''"` - SymbolCount int `gorm:"column:symbol_count;default:0"` - DecisionTF string `gorm:"column:decision_tf;default:''"` - ProcessedBars int `gorm:"column:processed_bars;default:0"` - ProgressPct float64 `gorm:"column:progress_pct;default:0"` - EquityLast float64 `gorm:"column:equity_last;default:0"` - MaxDrawdownPct float64 `gorm:"column:max_drawdown_pct;default:0"` - Liquidated bool `gorm:"column:liquidated;default:false"` - LiquidationNote string `gorm:"column:liquidation_note;default:''"` - PromptTemplate string `gorm:"column:prompt_template;default:''"` - CustomPrompt string `gorm:"column:custom_prompt;default:''"` - OverridePrompt bool `gorm:"column:override_prompt;default:false"` - AIProvider string `gorm:"column:ai_provider;default:''"` - AIModel string `gorm:"column:ai_model;default:''"` - LastError string `gorm:"column:last_error;default:''"` - CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` - UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` -} - -func (BacktestRun) TableName() string { - return "backtest_runs" -} - -// BacktestCheckpoint GORM model -type BacktestCheckpoint struct { - RunID string `gorm:"column:run_id;primaryKey"` - Payload []byte `gorm:"column:payload;not null"` - UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` -} - -func (BacktestCheckpoint) TableName() string { - return "backtest_checkpoints" -} - -// BacktestEquity GORM model -type BacktestEquity struct { - ID int64 `gorm:"primaryKey;autoIncrement"` - RunID string `gorm:"column:run_id;not null;index:idx_backtest_equity_run_ts"` - TS int64 `gorm:"column:ts;type:bigint;not null;index:idx_backtest_equity_run_ts"` - Equity float64 `gorm:"column:equity;not null"` - Available float64 `gorm:"column:available;not null"` - PnL float64 `gorm:"column:pnl;not null"` - PnLPct float64 `gorm:"column:pnl_pct;not null"` - DDPct float64 `gorm:"column:dd_pct;not null"` - Cycle int `gorm:"column:cycle;not null"` -} - -func (BacktestEquity) TableName() string { - return "backtest_equity" -} - -// BacktestTrade GORM model -type BacktestTrade struct { - ID int64 `gorm:"primaryKey;autoIncrement"` - RunID string `gorm:"column:run_id;not null;index:idx_backtest_trades_run_ts"` - TS int64 `gorm:"column:ts;type:bigint;not null;index:idx_backtest_trades_run_ts"` - Symbol string `gorm:"column:symbol;not null"` - Action string `gorm:"column:action;not null"` - Side string `gorm:"column:side;default:''"` - Qty float64 `gorm:"column:qty;default:0"` - Price float64 `gorm:"column:price;default:0"` - Fee float64 `gorm:"column:fee;default:0"` - Slippage float64 `gorm:"column:slippage;default:0"` - OrderValue float64 `gorm:"column:order_value;default:0"` - RealizedPnL float64 `gorm:"column:realized_pnl;default:0"` - Leverage int `gorm:"column:leverage;default:0"` - Cycle int `gorm:"column:cycle;default:0"` - PositionAfter float64 `gorm:"column:position_after;default:0"` - Liquidation bool `gorm:"column:liquidation;default:false"` - Note string `gorm:"column:note;default:''"` -} - -func (BacktestTrade) TableName() string { - return "backtest_trades" -} - -// BacktestMetrics GORM model -type BacktestMetrics struct { - RunID string `gorm:"column:run_id;primaryKey"` - Payload []byte `gorm:"column:payload;not null"` - UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"` -} - -func (BacktestMetrics) TableName() string { - return "backtest_metrics" -} - -// BacktestDecision GORM model -type BacktestDecision struct { - ID int64 `gorm:"primaryKey;autoIncrement"` - RunID string `gorm:"column:run_id;not null;index:idx_backtest_decisions_run_cycle"` - Cycle int `gorm:"column:cycle;not null;index:idx_backtest_decisions_run_cycle"` - Payload []byte `gorm:"column:payload;not null"` - CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"` -} - -func (BacktestDecision) TableName() string { - return "backtest_decisions" -} - -// initTables initializes backtest related tables -func (s *BacktestStore) initTables() error { - // For PostgreSQL with existing tables, skip AutoMigrate to avoid type conflicts - if s.db.Dialector.Name() == "postgres" { - var tableExists int64 - s.db.Raw(`SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'backtest_runs'`).Scan(&tableExists) - - if tableExists > 0 { - // Tables exist - fix column types and ensure indexes exist - // Fix ts column type from INTEGER to BIGINT (timestamps in milliseconds exceed int4 max) - s.db.Exec(`ALTER TABLE backtest_equity ALTER COLUMN ts TYPE BIGINT`) - s.db.Exec(`ALTER TABLE backtest_trades ALTER COLUMN ts TYPE BIGINT`) - s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_backtest_equity_run_ts ON backtest_equity(run_id, ts)`) - s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_backtest_trades_run_ts ON backtest_trades(run_id, ts)`) - s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_backtest_decisions_run_cycle ON backtest_decisions(run_id, cycle)`) - return nil - } - } - - // AutoMigrate all backtest tables - if err := s.db.AutoMigrate( - &BacktestRun{}, - &BacktestCheckpoint{}, - &BacktestEquity{}, - &BacktestTrade{}, - &BacktestMetrics{}, - &BacktestDecision{}, - ); err != nil { - return fmt.Errorf("failed to migrate backtest tables: %w", err) - } - - return nil -} - -// SaveCheckpoint saves checkpoint -func (s *BacktestStore) SaveCheckpoint(runID string, payload []byte) error { - checkpoint := BacktestCheckpoint{ - RunID: runID, - Payload: payload, - } - return s.db.Save(&checkpoint).Error -} - -// LoadCheckpoint loads checkpoint -func (s *BacktestStore) LoadCheckpoint(runID string) ([]byte, error) { - var checkpoint BacktestCheckpoint - err := s.db.Where("run_id = ?", runID).First(&checkpoint).Error - if err != nil { - return nil, err - } - return checkpoint.Payload, nil -} - -// SaveRunMetadata saves run metadata -func (s *BacktestStore) SaveRunMetadata(meta *RunMetadata) error { - run := BacktestRun{ - RunID: meta.RunID, - UserID: meta.UserID, - State: string(meta.State), - Label: meta.Label, - LastError: meta.LastError, - SymbolCount: meta.Summary.SymbolCount, - DecisionTF: meta.Summary.DecisionTF, - ProcessedBars: meta.Summary.ProcessedBars, - ProgressPct: meta.Summary.ProgressPct, - EquityLast: meta.Summary.EquityLast, - MaxDrawdownPct: meta.Summary.MaxDrawdownPct, - Liquidated: meta.Summary.Liquidated, - LiquidationNote: meta.Summary.LiquidationNote, - CreatedAt: meta.CreatedAt, - UpdatedAt: meta.UpdatedAt, - } - return s.db.Save(&run).Error -} - -// LoadRunMetadata loads run metadata -func (s *BacktestStore) LoadRunMetadata(runID string) (*RunMetadata, error) { - var run BacktestRun - err := s.db.Where("run_id = ?", runID).First(&run).Error - if err != nil { - return nil, err - } - - return &RunMetadata{ - RunID: run.RunID, - UserID: run.UserID, - Version: 1, - State: RunState(run.State), - Label: run.Label, - LastError: run.LastError, - Summary: RunSummary{ - SymbolCount: run.SymbolCount, - DecisionTF: run.DecisionTF, - ProcessedBars: run.ProcessedBars, - ProgressPct: run.ProgressPct, - EquityLast: run.EquityLast, - MaxDrawdownPct: run.MaxDrawdownPct, - Liquidated: run.Liquidated, - LiquidationNote: run.LiquidationNote, - }, - CreatedAt: run.CreatedAt, - UpdatedAt: run.UpdatedAt, - }, nil -} - -// ListRunIDs lists all run IDs -func (s *BacktestStore) ListRunIDs() ([]string, error) { - var runs []BacktestRun - err := s.db.Order("updated_at DESC").Find(&runs).Error - if err != nil { - return nil, err - } - - ids := make([]string, len(runs)) - for i, run := range runs { - ids[i] = run.RunID - } - return ids, nil -} - -// AppendEquityPoint appends equity point -func (s *BacktestStore) AppendEquityPoint(runID string, point EquityPoint) error { - eq := BacktestEquity{ - RunID: runID, - TS: point.Timestamp, - Equity: point.Equity, - Available: point.Available, - PnL: point.PnL, - PnLPct: point.PnLPct, - DDPct: point.DrawdownPct, - Cycle: point.Cycle, - } - return s.db.Create(&eq).Error -} - -// LoadEquityPoints loads equity points -func (s *BacktestStore) LoadEquityPoints(runID string) ([]EquityPoint, error) { - var eqs []BacktestEquity - err := s.db.Where("run_id = ?", runID).Order("ts ASC").Find(&eqs).Error - if err != nil { - return nil, err - } - - points := make([]EquityPoint, len(eqs)) - for i, eq := range eqs { - points[i] = EquityPoint{ - Timestamp: eq.TS, - Equity: eq.Equity, - Available: eq.Available, - PnL: eq.PnL, - PnLPct: eq.PnLPct, - DrawdownPct: eq.DDPct, - Cycle: eq.Cycle, - } - } - return points, nil -} - -// AppendTradeEvent appends trade event -func (s *BacktestStore) AppendTradeEvent(runID string, event TradeEvent) error { - trade := BacktestTrade{ - RunID: runID, - TS: event.Timestamp, - Symbol: event.Symbol, - Action: event.Action, - Side: event.Side, - Qty: event.Quantity, - Price: event.Price, - Fee: event.Fee, - Slippage: event.Slippage, - OrderValue: event.OrderValue, - RealizedPnL: event.RealizedPnL, - Leverage: event.Leverage, - Cycle: event.Cycle, - PositionAfter: event.PositionAfter, - Liquidation: event.LiquidationFlag, - Note: event.Note, - } - return s.db.Create(&trade).Error -} - -// LoadTradeEvents loads trade events -func (s *BacktestStore) LoadTradeEvents(runID string) ([]TradeEvent, error) { - var trades []BacktestTrade - err := s.db.Where("run_id = ?", runID).Order("ts ASC").Find(&trades).Error - if err != nil { - return nil, err - } - - events := make([]TradeEvent, len(trades)) - for i, trade := range trades { - events[i] = TradeEvent{ - Timestamp: trade.TS, - Symbol: trade.Symbol, - Action: trade.Action, - Side: trade.Side, - Quantity: trade.Qty, - Price: trade.Price, - Fee: trade.Fee, - Slippage: trade.Slippage, - OrderValue: trade.OrderValue, - RealizedPnL: trade.RealizedPnL, - Leverage: trade.Leverage, - Cycle: trade.Cycle, - PositionAfter: trade.PositionAfter, - LiquidationFlag: trade.Liquidation, - Note: trade.Note, - } - } - return events, nil -} - -// SaveMetrics saves metrics -func (s *BacktestStore) SaveMetrics(runID string, payload []byte) error { - metrics := BacktestMetrics{ - RunID: runID, - Payload: payload, - } - return s.db.Save(&metrics).Error -} - -// LoadMetrics loads metrics -func (s *BacktestStore) LoadMetrics(runID string) ([]byte, error) { - var metrics BacktestMetrics - err := s.db.Where("run_id = ?", runID).First(&metrics).Error - if err != nil { - return nil, err - } - return metrics.Payload, nil -} - -// SaveDecisionRecord saves decision record -func (s *BacktestStore) SaveDecisionRecord(runID string, cycle int, payload []byte) error { - decision := BacktestDecision{ - RunID: runID, - Cycle: cycle, - Payload: payload, - } - return s.db.Create(&decision).Error -} - -// LoadDecisionRecords loads decision records -func (s *BacktestStore) LoadDecisionRecords(runID string, limit, offset int) ([]json.RawMessage, error) { - var decisions []BacktestDecision - err := s.db.Where("run_id = ?", runID). - Order("id DESC"). - Limit(limit). - Offset(offset). - Find(&decisions).Error - if err != nil { - return nil, err - } - - records := make([]json.RawMessage, len(decisions)) - for i, d := range decisions { - records[i] = json.RawMessage(d.Payload) - } - return records, nil -} - -// LoadLatestDecision loads latest decision -func (s *BacktestStore) LoadLatestDecision(runID string, cycle int) ([]byte, error) { - var decision BacktestDecision - query := s.db.Where("run_id = ?", runID) - if cycle > 0 { - query = query.Where("cycle = ?", cycle) - } - err := query.Order("created_at DESC").First(&decision).Error - if err != nil { - return nil, err - } - return decision.Payload, nil -} - -// UpdateProgress updates progress -func (s *BacktestStore) UpdateProgress(runID string, progressPct, equity float64, barIndex int, liquidated bool) error { - return s.db.Model(&BacktestRun{}).Where("run_id = ?", runID).Updates(map[string]interface{}{ - "progress_pct": progressPct, - "equity_last": equity, - "processed_bars": barIndex, - "liquidated": liquidated, - }).Error -} - -// ListIndexEntries lists index entries -func (s *BacktestStore) ListIndexEntries() ([]RunIndexEntry, error) { - var runs []BacktestRun - err := s.db.Order("updated_at DESC").Find(&runs).Error - if err != nil { - return nil, err - } - - entries := make([]RunIndexEntry, len(runs)) - for i, run := range runs { - entry := RunIndexEntry{ - RunID: run.RunID, - State: run.State, - DecisionTF: run.DecisionTF, - EquityLast: run.EquityLast, - MaxDrawdownPct: run.MaxDrawdownPct, - CreatedAtISO: run.CreatedAt.Format(time.RFC3339), - UpdatedAtISO: run.UpdatedAt.Format(time.RFC3339), - Symbols: make([]string, 0, run.SymbolCount), - } - - if len(run.ConfigJSON) > 0 { - var cfg struct { - Symbols []string `json:"symbols"` - StartTS int64 `json:"start_ts"` - EndTS int64 `json:"end_ts"` - } - if json.Unmarshal(run.ConfigJSON, &cfg) == nil { - entry.Symbols = cfg.Symbols - entry.StartTS = cfg.StartTS - entry.EndTS = cfg.EndTS - } - } - - entries[i] = entry - } - return entries, nil -} - -// DeleteRun deletes run -func (s *BacktestStore) DeleteRun(runID string) error { - // Delete related records first (cascade may not work in all cases) - s.db.Where("run_id = ?", runID).Delete(&BacktestCheckpoint{}) - s.db.Where("run_id = ?", runID).Delete(&BacktestEquity{}) - s.db.Where("run_id = ?", runID).Delete(&BacktestTrade{}) - s.db.Where("run_id = ?", runID).Delete(&BacktestMetrics{}) - s.db.Where("run_id = ?", runID).Delete(&BacktestDecision{}) - - return s.db.Where("run_id = ?", runID).Delete(&BacktestRun{}).Error -} - -// SaveConfig saves config -func (s *BacktestStore) SaveConfig(runID, userID, template, customPrompt, provider, model string, override bool, configJSON []byte) error { - if userID == "" { - userID = "default" - } - - run := BacktestRun{ - RunID: runID, - UserID: userID, - ConfigJSON: configJSON, - PromptTemplate: template, - CustomPrompt: customPrompt, - OverridePrompt: override, - AIProvider: provider, - AIModel: model, - } - return s.db.Save(&run).Error -} - -// LoadConfig loads config -func (s *BacktestStore) LoadConfig(runID string) ([]byte, error) { - var run BacktestRun - err := s.db.Where("run_id = ?", runID).First(&run).Error - if err != nil { - return nil, err - } - return run.ConfigJSON, nil -} diff --git a/store/store.go b/store/store.go index 5e6ec457..9cd1645e 100644 --- a/store/store.go +++ b/store/store.go @@ -23,7 +23,6 @@ type Store struct { exchange *ExchangeStore trader *TraderStore decision *DecisionStore - backtest *BacktestStore position *PositionStore strategy *StrategyStore equity *EquityStore @@ -143,9 +142,6 @@ func (s *Store) initTables() error { if err := s.Decision().initTables(); err != nil { return fmt.Errorf("failed to initialize decision log tables: %w", err) } - if err := s.Backtest().initTables(); err != nil { - return fmt.Errorf("failed to initialize backtest tables: %w", err) - } if err := s.Position().InitTables(); err != nil { return fmt.Errorf("failed to initialize position tables: %w", err) } @@ -237,16 +233,6 @@ func (s *Store) Decision() *DecisionStore { return s.decision } -// Backtest gets backtest data storage -func (s *Store) Backtest() *BacktestStore { - s.mu.Lock() - defer s.mu.Unlock() - if s.backtest == nil { - s.backtest = NewBacktestStore(s.gdb) - } - return s.backtest -} - // Position gets position storage func (s *Store) Position() *PositionStore { s.mu.Lock() diff --git a/web/src/App.tsx b/web/src/App.tsx index 01d497db..17a9c2b1 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -24,7 +24,6 @@ import { t } from './i18n/translations' import { useSystemConfig } from './hooks/useSystemConfig' import { OFFICIAL_LINKS } from './constants/branding' -import { BacktestPage } from './components/backtest/BacktestPage' import type { SystemStatus, AccountInfo, @@ -39,7 +38,6 @@ type Page = | 'competition' | 'traders' | 'trader' - | 'backtest' | 'strategy' | 'strategy-market' | 'data' @@ -66,7 +64,6 @@ function App() { const hash = window.location.hash.slice(1) // 去掉 # if (path === '/traders' || hash === 'traders') return 'traders' - if (path === '/backtest' || hash === 'backtest') return 'backtest' if (path === '/strategy' || hash === 'strategy') return 'strategy' if (path === '/strategy-market' || hash === 'strategy-market') return 'strategy-market' if (path === '/data' || hash === 'data') return 'data' @@ -92,7 +89,6 @@ function App() { 'data': '/data', 'traders': '/traders', 'trader': '/dashboard', - 'backtest': '/backtest', 'strategy': '/strategy', 'faq': '/faq', 'login': '/login', @@ -147,8 +143,6 @@ function App() { if (path === '/traders' || hash === 'traders') { setCurrentPage('traders') - } else if (path === '/backtest' || hash === 'backtest') { - setCurrentPage('backtest') } else if (path === '/strategy' || hash === 'strategy') { setCurrentPage('strategy') } else if (path === '/strategy-market' || hash === 'strategy-market') { @@ -410,7 +404,6 @@ function App() { 'strategy-market': '/strategy-market', 'traders': '/traders', 'trader': '/dashboard', - 'backtest': '/backtest', 'strategy': '/strategy', 'faq': '/faq', } @@ -496,8 +489,6 @@ function App() { setCurrentPage('trader') }} /> - ) : currentPage === 'backtest' ? ( - ) : currentPage === 'strategy' ? ( ) : ( diff --git a/web/src/components/backtest/BacktestChartTab.tsx b/web/src/components/backtest/BacktestChartTab.tsx deleted file mode 100644 index 80b0408a..00000000 --- a/web/src/components/backtest/BacktestChartTab.tsx +++ /dev/null @@ -1,434 +0,0 @@ -import { useEffect, useMemo, useState, useRef } from 'react' -import { motion } from 'framer-motion' -import { - createChart, - ColorType, - CrosshairMode, - CandlestickSeries, - createSeriesMarkers, - type IChartApi, - type ISeriesApi, - type CandlestickData, - type UTCTimestamp, - type SeriesMarker, -} from 'lightweight-charts' -import { - ResponsiveContainer, - AreaChart, - Area, - XAxis, - YAxis, - CartesianGrid, - Tooltip, - ReferenceDot, -} from 'recharts' -import { - Clock, - AlertTriangle, - RefreshCw, - CandlestickChart as CandlestickIcon, -} from 'lucide-react' -import { api } from '../../lib/api' -import { t, type Language } from '../../i18n/translations' -import type { - BacktestEquityPoint, - BacktestTradeEvent, - BacktestKlinesResponse, -} from '../../types' - -// ============ Equity Chart (Recharts) ============ - -interface EquityChartProps { - equity: BacktestEquityPoint[] - trades: BacktestTradeEvent[] -} - -export function EquityChart({ equity, trades }: EquityChartProps) { - const chartData = useMemo(() => { - return equity.map((point) => ({ - time: new Date(point.ts).toLocaleString(), - ts: point.ts, - equity: point.equity, - pnl_pct: point.pnl_pct, - })) - }, [equity]) - - const tradeMarkers = useMemo(() => { - if (!trades.length || !equity.length) return [] - return trades - .filter((t) => t.action.includes('open') || t.action.includes('close')) - .map((trade) => { - const closest = equity.reduce((prev, curr) => - Math.abs(curr.ts - trade.ts) < Math.abs(prev.ts - trade.ts) ? curr : prev - ) - return { - ts: closest.ts, - equity: closest.equity, - action: trade.action, - symbol: trade.symbol, - isOpen: trade.action.includes('open'), - } - }) - .slice(-30) - }, [trades, equity]) - - return ( -
- - - - - - - - - - - - [`$${value.toFixed(2)}`, 'Equity']} - /> - - {tradeMarkers.map((marker, idx) => ( - d.ts === marker.ts)} - y={marker.equity} - r={4} - fill={marker.isOpen ? '#0ECB81' : '#F6465D'} - stroke={marker.isOpen ? '#0ECB81' : '#F6465D'} - /> - ))} - - -
- ) -} - -// ============ Candlestick Chart with Trade Markers ============ - -interface CandlestickChartProps { - runId: string - trades: BacktestTradeEvent[] - language: Language -} - -export function CandlestickChartComponent({ runId, trades, language }: CandlestickChartProps) { - const chartContainerRef = useRef(null) - const chartRef = useRef(null) - const candleSeriesRef = useRef | null>(null) - - const symbols = useMemo(() => { - const symbolSet = new Set(trades.map((t) => t.symbol)) - return Array.from(symbolSet).sort() - }, [trades]) - - const [selectedSymbol, setSelectedSymbol] = useState(symbols[0] || '') - const [selectedTimeframe, setSelectedTimeframe] = useState('15m') - const [isLoading, setIsLoading] = useState(false) - const [error, setError] = useState(null) - - const CHART_TIMEFRAMES = ['1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d'] - - useEffect(() => { - if (symbols.length > 0 && !symbols.includes(selectedSymbol)) { - setSelectedSymbol(symbols[0]) - } - }, [symbols, selectedSymbol]) - - const symbolTrades = useMemo(() => { - return trades.filter((t) => t.symbol === selectedSymbol) - }, [trades, selectedSymbol]) - - useEffect(() => { - if (!chartContainerRef.current || !selectedSymbol || !runId) return - - const container = chartContainerRef.current - - const chart = createChart(container, { - layout: { - background: { type: ColorType.Solid, color: '#0B0E11' }, - textColor: '#848E9C', - }, - grid: { - vertLines: { color: 'rgba(43, 49, 57, 0.5)' }, - horzLines: { color: 'rgba(43, 49, 57, 0.5)' }, - }, - crosshair: { - mode: CrosshairMode.Normal, - }, - rightPriceScale: { - borderColor: '#2B3139', - }, - timeScale: { - borderColor: '#2B3139', - timeVisible: true, - secondsVisible: false, - }, - width: container.clientWidth, - height: 400, - }) - - chartRef.current = chart - - const candleSeries = chart.addSeries(CandlestickSeries, { - upColor: '#0ECB81', - downColor: '#F6465D', - borderUpColor: '#0ECB81', - borderDownColor: '#F6465D', - wickUpColor: '#0ECB81', - wickDownColor: '#F6465D', - }) - candleSeriesRef.current = candleSeries - - setIsLoading(true) - setError(null) - - api - .getBacktestKlines(runId, selectedSymbol, selectedTimeframe) - .then((data: BacktestKlinesResponse) => { - const klineData: CandlestickData[] = data.klines.map((k) => ({ - time: k.time as UTCTimestamp, - open: k.open, - high: k.high, - low: k.low, - close: k.close, - })) - candleSeries.setData(klineData) - - const markers: SeriesMarker[] = symbolTrades - .map((trade) => { - const tradeTime = Math.floor(trade.ts / 1000) - const closestKline = data.klines.reduce((prev, curr) => - Math.abs(curr.time - tradeTime) < Math.abs(prev.time - tradeTime) ? curr : prev - ) - const isOpen = trade.action.includes('open') - const isLong = trade.side === 'long' || trade.action.includes('long') - const pnl = trade.realized_pnl - - let text = '' - let color = '#0ECB81' - - if (isOpen) { - if (isLong) { - text = `▲ Long @${trade.price.toFixed(2)}` - color = '#0ECB81' - } else { - text = `▼ Short @${trade.price.toFixed(2)}` - color = '#F6465D' - } - } else { - const pnlStr = pnl >= 0 ? `+$${pnl.toFixed(2)}` : `-$${Math.abs(pnl).toFixed(2)}` - text = `✕ ${pnlStr}` - color = pnl >= 0 ? '#0ECB81' : '#F6465D' - } - - return { - time: closestKline.time as UTCTimestamp, - position: isOpen - ? (isLong ? 'belowBar' as const : 'aboveBar' as const) - : (isLong ? 'aboveBar' as const : 'belowBar' as const), - color, - shape: 'circle' as const, - size: 2, - text, - } - }) - .sort((a, b) => (a.time as number) - (b.time as number)) - - createSeriesMarkers(candleSeries, markers) - chart.timeScale().fitContent() - setIsLoading(false) - }) - .catch((err) => { - setError(err.message || 'Failed to load klines') - setIsLoading(false) - }) - - const handleResize = () => { - if (chartContainerRef.current) { - chart.applyOptions({ width: chartContainerRef.current.clientWidth }) - } - } - window.addEventListener('resize', handleResize) - - return () => { - window.removeEventListener('resize', handleResize) - chart.remove() - chartRef.current = null - candleSeriesRef.current = null - } - }, [runId, selectedSymbol, selectedTimeframe, symbolTrades]) - - if (symbols.length === 0) { - return ( -
- {t('backtestChart.noTrades', language)} -
- ) - } - - return ( -
-
-
- - - {t('backtestChart.symbol', language)} - - -
- -
- - - {t('backtestChart.interval', language)} - -
- {CHART_TIMEFRAMES.map((tf) => ( - - ))} -
-
- - - ({symbolTrades.length} {t('backtestChart.trades', language)}) - -
- -
- {isLoading && ( -
- - {t('backtestChart.loadingKline', language)} -
- )} - {error && ( -
- - {error} -
- )} -
- -
-
-
- {t('backtestChart.openProfit', language)} -
-
-
- {t('backtestChart.lossClose', language)} -
- | - ▲ Long · ▼ Short · ✕ {t('backtestChart.close', language)} -
-
- ) -} - -// ============ Chart Tab Content ============ - -interface BacktestChartTabProps { - equity: BacktestEquityPoint[] | undefined - trades: BacktestTradeEvent[] | undefined - selectedRunId: string - language: Language - tr: (key: string) => string -} - -export function BacktestChartTab({ - equity, - trades, - selectedRunId, - language, - tr, -}: BacktestChartTabProps) { - return ( - -
-

- {t('backtestChart.equityCurve', language)} -

- {equity && equity.length > 0 ? ( - - ) : ( -
- {tr('charts.equityEmpty')} -
- )} -
- - {selectedRunId && trades && trades.length > 0 && ( -
-

- {t('backtestChart.candlestickTradeMarkers', language)} -

- -
- )} - - ) -} diff --git a/web/src/components/backtest/BacktestConfigForm.tsx b/web/src/components/backtest/BacktestConfigForm.tsx deleted file mode 100644 index cbfb73ad..00000000 --- a/web/src/components/backtest/BacktestConfigForm.tsx +++ /dev/null @@ -1,597 +0,0 @@ -import { useMemo, type FormEvent } from 'react' -import { motion, AnimatePresence } from 'framer-motion' -import { - ChevronRight, - ChevronLeft, - RefreshCw, - Zap, -} from 'lucide-react' -import type { AIModel, Strategy } from '../../types' -import { t as globalT } from '../../i18n/translations' -import type { Language } from '../../i18n/translations' - -// ============ Types ============ - -type WizardStep = 1 | 2 | 3 - -export interface BacktestFormState { - runId: string - symbols: string - timeframes: string[] - decisionTf: string - cadence: number - start: string - end: string - balance: number - fee: number - slippage: number - btcEthLeverage: number - altcoinLeverage: number - fill: string - prompt: string - promptTemplate: string - customPrompt: string - overridePrompt: boolean - cacheAI: boolean - replayOnly: boolean - aiModelId: string - strategyId: string -} - -const TIMEFRAME_OPTIONS = ['1m', '3m', '5m', '15m', '30m', '1h', '4h', '1d'] -const POPULAR_SYMBOLS = ['BTCUSDT', 'ETHUSDT', 'SOLUSDT', 'BNBUSDT', 'XRPUSDT', 'DOGEUSDT'] - -// ============ Config Form ============ - -interface BacktestConfigFormProps { - formState: BacktestFormState - wizardStep: WizardStep - isStarting: boolean - aiModels: AIModel[] | undefined - strategies: Strategy[] | undefined - language: string - tr: (key: string, params?: Record) => string - onFormChange: (key: string, value: string | number | boolean | string[]) => void - onWizardStepChange: (step: WizardStep) => void - onStart: (event: FormEvent) => void -} - -export function BacktestConfigForm({ - formState, - wizardStep, - isStarting, - aiModels, - strategies, - language, - tr, - onFormChange, - onWizardStepChange, - onStart, -}: BacktestConfigFormProps) { - const selectedModel = aiModels?.find((m) => m.id === formState.aiModelId) - const selectedStrategy = strategies?.find((s) => s.id === formState.strategyId) - - const strategyHasDynamicCoins = useMemo(() => { - const cs = selectedStrategy?.config?.coin_source - if (!cs) return false - const st = cs.source_type as string - if (st === 'ai500' || st === 'oi_top') return true - if (st === 'mixed' && (cs.use_ai500 || cs.use_oi_top)) return true - if (!st && (cs.use_ai500 || cs.use_oi_top)) return true - return false - }, [selectedStrategy]) - - const coinSourceDescription = useMemo(() => { - const cs = selectedStrategy?.config?.coin_source - if (!cs) return null - let st = cs.source_type as string - if (!st) { - if (cs.use_ai500 && cs.use_oi_top) st = 'mixed' - else if (cs.use_ai500) st = 'ai500' - else if (cs.use_oi_top) st = 'oi_top' - else if (cs.static_coins?.length) st = 'static' - } - switch (st) { - case 'ai500': return { type: 'AI500', limit: cs.ai500_limit || 30 } - case 'oi_top': return { type: 'OI Top', limit: cs.oi_top_limit || 30 } - case 'mixed': { - const parts: string[] = [] - if (cs.use_ai500) parts.push(`AI500(${cs.ai500_limit || 30})`) - if (cs.use_oi_top) parts.push(`OI Top(${cs.oi_top_limit || 30})`) - if (cs.static_coins?.length) parts.push(`Static(${cs.static_coins.length})`) - return { type: 'Mixed', desc: parts.join(' + ') } - } - case 'static': return { type: 'Static', coins: cs.static_coins || [] } - default: return null - } - }, [selectedStrategy]) - - const lang = language as Language - const quickRanges = [ - { label: globalT('backtestConfigForm.quickRange24h', lang), hours: 24 }, - { label: globalT('backtestConfigForm.quickRange3d', lang), hours: 72 }, - { label: globalT('backtestConfigForm.quickRange7d', lang), hours: 168 }, - { label: globalT('backtestConfigForm.quickRange30d', lang), hours: 720 }, - ] - - const applyQuickRange = (hours: number) => { - const end = new Date() - const start = new Date(end.getTime() - hours * 3600 * 1000) - const fmt = (d: Date) => new Date(d.getTime() - d.getTimezoneOffset() * 60000).toISOString().slice(0, 16) - onFormChange('start', fmt(start)) - onFormChange('end', fmt(end)) - } - - return ( -
-
- {[1, 2, 3].map((step) => ( -
- - {step < 3 && ( -
step ? '#F0B90B' : '#2B3139' }} - /> - )} -
- ))} - - {wizardStep === 1 ? globalT('backtestConfigForm.selectModel', lang) - : wizardStep === 2 ? globalT('backtestConfigForm.configure', lang) - : globalT('backtestConfigForm.confirmStart', lang)} - -
- -
- - {/* Step 1: Model & Symbols */} - {wizardStep === 1 && ( - -
- - - {selectedModel && ( -
- - {selectedModel.enabled ? tr('form.enabled') : tr('form.disabled')} - -
- )} -
- - {/* Strategy Selection (Optional) */} -
- - - {formState.strategyId && coinSourceDescription && ( -
-
- - {globalT('backtestConfigForm.coinSource', lang)} - - - {coinSourceDescription.type} - {coinSourceDescription.limit && ` (${coinSourceDescription.limit})`} - {coinSourceDescription.desc && ` - ${coinSourceDescription.desc}`} - -
- {strategyHasDynamicCoins && ( -
- {globalT('backtestConfigForm.clearDynamicCoins', lang)} -
- )} -
- )} -
- -
- - {!strategyHasDynamicCoins && ( -
- {POPULAR_SYMBOLS.map((sym) => { - const isSelected = formState.symbols.includes(sym) - return ( - - ) - })} -
- )} -
-