reliability: wrap all 27 bare goroutines with safe.Go/GoNamed panic recovery

Applied safe.GoNamed to:
- 9 exchange order_sync goroutines (OKX, Hyperliquid, Aster, Bybit, KuCoin, Gate, Bitget, Lighter, Binance×2)
- Drawdown monitor (auto_trader_risk.go)
- Brain news scanner + market briefs (agent/brain.go)
- Sentinel scanner (agent/sentinel.go)
- Agent scheduler (agent/scheduler.go)
- x402 idle watchdog (mcp/payment/x402.go)
- MCP stream idle watchdog (mcp/client.go)
- Rate limiter cleanup (api/rate_limiter.go)
- 3 telemetry fire-and-forget sends (telemetry/experience.go)
- CoinAnk WS handler (provider/coinank/coinank_api/kline_ws.go)
- API server goroutine (main.go)

Added manual defer/recover with error reporting to:
- Telegram AI agent handler (sends error msg to user on panic)
- Trader data fetch (returns error result on panic to prevent deadlock)

Before: a panic in ANY of these 27 goroutines would crash the entire
trading process with zero diagnostics. Now all panics are caught, logged
with stack traces, and the process continues running.
This commit is contained in:
shinchan-zhai
2026-03-23 10:30:13 +08:00
parent 07b4f30b85
commit 5e06037fa2
21 changed files with 84 additions and 48 deletions

View File

@@ -3,6 +3,7 @@ package aster
import (
"fmt"
"nofx/logger"
"nofx/safe"
"nofx/market"
"nofx/store"
"sort"
@@ -182,12 +183,12 @@ func deriveAsterOrderAction(side, positionSide string, realizedPnL float64) stri
// StartOrderSync starts background order sync task for Aster
func (t *AsterTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("aster-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromAster(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Aster order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Aster order sync started (interval: %v)", interval)
}

View File

@@ -3,6 +3,7 @@ package trader
import (
"fmt"
"nofx/logger"
"nofx/safe"
"strings"
"time"
)
@@ -10,7 +11,7 @@ import (
// startDrawdownMonitor starts drawdown monitoring
func (at *AutoTrader) startDrawdownMonitor() {
at.monitorWg.Add(1)
go func() {
safe.GoNamed("drawdown-monitor", func() {
defer at.monitorWg.Done()
ticker := time.NewTicker(1 * time.Minute) // Check every minute
@@ -27,7 +28,7 @@ func (at *AutoTrader) startDrawdownMonitor() {
return
}
}
}()
})
}
// checkPositionDrawdown checks position drawdown situation

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"nofx/logger"
"nofx/market"
"nofx/safe"
"nofx/store"
"nofx/trader/types"
"sort"
@@ -351,21 +352,21 @@ func (t *FuturesTrader) determineOrderAction(side, positionSide string, realized
// StartOrderSync starts background order sync task for Binance
func (t *FuturesTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
// Run first sync immediately
go func() {
safe.GoNamed("binance-order-sync-initial", func() {
logger.Infof("🔄 Running initial Binance order sync...")
if err := t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Initial Binance order sync failed: %v", err)
}
}()
})
// Then run periodically
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("binance-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Binance order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Binance order sync started (interval: %v)", interval)
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"nofx/logger"
"nofx/market"
"nofx/safe"
"nofx/store"
"sort"
"strconv"
@@ -281,12 +282,12 @@ func (t *BitgetTrader) SyncOrdersFromBitget(traderID string, exchangeID string,
// StartOrderSync starts background order sync task for Bitget
func (t *BitgetTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("bitget-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromBitget(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Bitget order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Bitget order sync started (interval: %v)", interval)
}

View File

@@ -10,6 +10,7 @@ import (
"net/http"
"nofx/logger"
"nofx/market"
"nofx/safe"
"nofx/store"
"sort"
"strconv"
@@ -300,12 +301,12 @@ func (t *BybitTrader) SyncOrdersFromBybit(traderID string, exchangeID string, ex
// StartOrderSync starts background order sync task for Bybit
func (t *BybitTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("bybit-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromBybit(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Bybit order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Bybit order sync started (interval: %v)", interval)
}

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"nofx/logger"
"nofx/market"
"nofx/safe"
"nofx/store"
"sort"
"strconv"
@@ -293,12 +294,12 @@ func (t *GateTrader) SyncOrdersFromGate(traderID string, exchangeID string, exch
// StartOrderSync starts background order sync task for Gate
func (t *GateTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("gate-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromGate(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Gate order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Gate order sync started (interval: %v)", interval)
}

View File

@@ -3,6 +3,7 @@ package hyperliquid
import (
"fmt"
"nofx/logger"
"nofx/safe"
"nofx/market"
"nofx/store"
"sort"
@@ -138,12 +139,12 @@ func (t *HyperliquidTrader) SyncOrdersFromHyperliquid(traderID string, exchangeI
// StartOrderSync starts background order sync task
func (t *HyperliquidTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("hyperliquid-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromHyperliquid(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Hyperliquid order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 Hyperliquid order sync started (interval: %v)", interval)
}

View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"nofx/logger"
"nofx/safe"
"nofx/store"
"nofx/trader/types"
"sort"
@@ -401,12 +402,12 @@ func (t *KuCoinTrader) SyncOrdersFromKuCoin(traderID string, exchangeID string,
// StartOrderSync starts background order sync task for KuCoin
func (t *KuCoinTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("kucoin-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromKuCoin(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ KuCoin order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 KuCoin order sync started (interval: %v)", interval)
}

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"nofx/logger"
"nofx/market"
"nofx/safe"
"nofx/store"
"sort"
"strings"
@@ -147,7 +148,7 @@ func (t *LighterTraderV2) SyncOrdersFromLighter(traderID string, exchangeID stri
// StartOrderSync starts background order sync task
func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("lighter-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromLighter(traderID, exchangeID, exchangeType, st); err != nil {
// Only log non-404 errors to reduce log spam
@@ -156,6 +157,6 @@ func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exc
}
}
}
}()
})
logger.Infof("🔄 Lighter order+position sync started (interval: %v)", interval)
}

View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"nofx/logger"
"nofx/safe"
"nofx/market"
"nofx/store"
"sort"
@@ -274,12 +275,12 @@ func (t *OKXTrader) SyncOrdersFromOKX(traderID string, exchangeID string, exchan
// StartOrderSync starts background order sync task for OKX
func (t *OKXTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) {
ticker := time.NewTicker(interval)
go func() {
safe.GoNamed("okx-order-sync", func() {
for range ticker.C {
if err := t.SyncOrdersFromOKX(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ OKX order sync failed: %v", err)
}
}
}()
})
logger.Infof("🔄 OKX order sync started (interval: %v)", interval)
}