mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-07 13:00:59 +08:00
reliability: wrap all 27 bare goroutines with safe.Go/GoNamed panic recovery
Applied safe.GoNamed to: - 9 exchange order_sync goroutines (OKX, Hyperliquid, Aster, Bybit, KuCoin, Gate, Bitget, Lighter, Binance×2) - Drawdown monitor (auto_trader_risk.go) - Brain news scanner + market briefs (agent/brain.go) - Sentinel scanner (agent/sentinel.go) - Agent scheduler (agent/scheduler.go) - x402 idle watchdog (mcp/payment/x402.go) - MCP stream idle watchdog (mcp/client.go) - Rate limiter cleanup (api/rate_limiter.go) - 3 telemetry fire-and-forget sends (telemetry/experience.go) - CoinAnk WS handler (provider/coinank/coinank_api/kline_ws.go) - API server goroutine (main.go) Added manual defer/recover with error reporting to: - Telegram AI agent handler (sends error msg to user on panic) - Trader data fetch (returns error result on panic to prevent deadlock) Before: a panic in ANY of these 27 goroutines would crash the entire trading process with zero diagnostics. Now all panics are caught, logged with stack traces, and the process continues running.
This commit is contained in:
@@ -3,6 +3,7 @@ package aster
|
||||
import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/safe"
|
||||
"nofx/market"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
@@ -182,12 +183,12 @@ func deriveAsterOrderAction(side, positionSide string, realizedPnL float64) stri
|
||||
// StartOrderSync starts background order sync task for Aster
|
||||
func (t *AsterTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("aster-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromAster(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Aster order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Aster order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package trader
|
||||
import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/safe"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -10,7 +11,7 @@ import (
|
||||
// startDrawdownMonitor starts drawdown monitoring
|
||||
func (at *AutoTrader) startDrawdownMonitor() {
|
||||
at.monitorWg.Add(1)
|
||||
go func() {
|
||||
safe.GoNamed("drawdown-monitor", func() {
|
||||
defer at.monitorWg.Done()
|
||||
|
||||
ticker := time.NewTicker(1 * time.Minute) // Check every minute
|
||||
@@ -27,7 +28,7 @@ func (at *AutoTrader) startDrawdownMonitor() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// checkPositionDrawdown checks position drawdown situation
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"nofx/trader/types"
|
||||
"sort"
|
||||
@@ -351,21 +352,21 @@ func (t *FuturesTrader) determineOrderAction(side, positionSide string, realized
|
||||
// StartOrderSync starts background order sync task for Binance
|
||||
func (t *FuturesTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
// Run first sync immediately
|
||||
go func() {
|
||||
safe.GoNamed("binance-order-sync-initial", func() {
|
||||
logger.Infof("🔄 Running initial Binance order sync...")
|
||||
if err := t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Initial Binance order sync failed: %v", err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
|
||||
// Then run periodically
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("binance-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Binance order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Binance order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -281,12 +282,12 @@ func (t *BitgetTrader) SyncOrdersFromBitget(traderID string, exchangeID string,
|
||||
// StartOrderSync starts background order sync task for Bitget
|
||||
func (t *BitgetTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("bitget-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromBitget(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Bitget order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Bitget order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -300,12 +301,12 @@ func (t *BybitTrader) SyncOrdersFromBybit(traderID string, exchangeID string, ex
|
||||
// StartOrderSync starts background order sync task for Bybit
|
||||
func (t *BybitTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("bybit-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromBybit(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Bybit order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Bybit order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -293,12 +294,12 @@ func (t *GateTrader) SyncOrdersFromGate(traderID string, exchangeID string, exch
|
||||
// StartOrderSync starts background order sync task for Gate
|
||||
func (t *GateTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("gate-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromGate(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Gate order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Gate order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package hyperliquid
|
||||
import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/safe"
|
||||
"nofx/market"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
@@ -138,12 +139,12 @@ func (t *HyperliquidTrader) SyncOrdersFromHyperliquid(traderID string, exchangeI
|
||||
// StartOrderSync starts background order sync task
|
||||
func (t *HyperliquidTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("hyperliquid-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromHyperliquid(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ Hyperliquid order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Hyperliquid order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"nofx/trader/types"
|
||||
"sort"
|
||||
@@ -401,12 +402,12 @@ func (t *KuCoinTrader) SyncOrdersFromKuCoin(traderID string, exchangeID string,
|
||||
// StartOrderSync starts background order sync task for KuCoin
|
||||
func (t *KuCoinTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("kucoin-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromKuCoin(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ KuCoin order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 KuCoin order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/safe"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -147,7 +148,7 @@ func (t *LighterTraderV2) SyncOrdersFromLighter(traderID string, exchangeID stri
|
||||
// StartOrderSync starts background order sync task
|
||||
func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("lighter-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromLighter(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
// Only log non-404 errors to reduce log spam
|
||||
@@ -156,6 +157,6 @@ func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exc
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 Lighter order+position sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"nofx/logger"
|
||||
"nofx/safe"
|
||||
"nofx/market"
|
||||
"nofx/store"
|
||||
"sort"
|
||||
@@ -274,12 +275,12 @@ func (t *OKXTrader) SyncOrdersFromOKX(traderID string, exchangeID string, exchan
|
||||
// StartOrderSync starts background order sync task for OKX
|
||||
func (t *OKXTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
safe.GoNamed("okx-order-sync", func() {
|
||||
for range ticker.C {
|
||||
if err := t.SyncOrdersFromOKX(traderID, exchangeID, exchangeType, st); err != nil {
|
||||
logger.Infof("⚠️ OKX order sync failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
logger.Infof("🔄 OKX order sync started (interval: %v)", interval)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user