fix(trader): stop order-sync goroutine leak and rate-limit hammering

Every StartOrderSync spawned a ticker goroutine that ran forever — it
survived trader stop AND deletion, so each quick-created trader left a
permanent 30s Hyperliquid poll behind. Stacked leaks turned into an
~8s effective hammer that tripped Hyperliquid's 429 rate limit, which
then broke the symbol board, trader creation, and order sync itself.

- new trader/syncloop package: shared stoppable sync loop with
  exponential failure backoff (30s base, 5min cap)
- all 9 exchanges' StartOrderSync now take the trader's stop channel
  and stop when the trader stops (close broadcast from AutoTrader.Stop)
- provider/hyperliquid: GetPerpDexCoins now serves a 5min TTL cache and
  falls back to the stale board when the upstream returns 429, so the
  symbol panel keeps working through rate limiting
This commit is contained in:
tinkle-community
2026-06-11 21:45:31 +08:00
parent 133ef51de8
commit 953240565f
14 changed files with 424 additions and 178 deletions

View File

@@ -164,9 +164,59 @@ func fetchPerpDexCoins(ctx context.Context, client *http.Client, dex string) ([]
return coins, nil return coins, nil
} }
// GetPerpDexCoins fetches current tradable USDC perp assets for a given Hyperliquid dex. // perpDexCacheTTL bounds how often the perp-dex symbol board is re-fetched.
// The tradable symbol list changes rarely; prices/volume on the board are
// display hints, so short staleness is far better than hammering the
// Hyperliquid API (which rate-limits with 429) on every panel render.
const perpDexCacheTTL = 5 * time.Minute
type perpDexCacheEntry struct {
coins []CoinInfo
fetchedAt time.Time
}
type perpDexCacheStore struct {
mu sync.Mutex
entries map[string]perpDexCacheEntry
}
var perpDexCoinCache = &perpDexCacheStore{entries: map[string]perpDexCacheEntry{}}
// fetchPerpDexCoinsFn is swappable in tests.
var fetchPerpDexCoinsFn = fetchPerpDexCoins
// GetPerpDexCoins returns current tradable USDC perp assets for a given
// Hyperliquid dex, served from a TTL cache. When the upstream fetch fails
// (e.g. HTTP 429 rate limiting) and stale data exists, the stale board is
// served instead of an error so the UI keeps working.
func GetPerpDexCoins(ctx context.Context, dex string) ([]CoinInfo, error) { func GetPerpDexCoins(ctx context.Context, dex string) ([]CoinInfo, error) {
return fetchPerpDexCoins(ctx, &http.Client{Timeout: 30 * time.Second}, dex) perpDexCoinCache.mu.Lock()
defer perpDexCoinCache.mu.Unlock()
entry, hasCache := perpDexCoinCache.entries[dex]
if hasCache && time.Since(entry.fetchedAt) < perpDexCacheTTL {
return copyCoins(entry.coins), nil
}
coins, err := fetchPerpDexCoinsFn(ctx, &http.Client{Timeout: 30 * time.Second}, dex)
if err != nil {
if hasCache {
logger.Infof("⚠️ Hyperliquid perp-dex fetch failed (%v); serving cached board for dex %q from %s",
err, dex, entry.fetchedAt.Format(time.RFC3339))
return copyCoins(entry.coins), nil
}
return nil, err
}
perpDexCoinCache.entries[dex] = perpDexCacheEntry{coins: coins, fetchedAt: time.Now()}
return copyCoins(coins), nil
}
// copyCoins returns a defensive copy so callers cannot mutate the cache.
func copyCoins(coins []CoinInfo) []CoinInfo {
out := make([]CoinInfo, len(coins))
copy(out, coins)
return out
} }
// fetchCoins fetches all default Hyperliquid crypto coins and sorts by volume // fetchCoins fetches all default Hyperliquid crypto coins and sorts by volume

View File

@@ -0,0 +1,113 @@
package hyperliquid
import (
"context"
"errors"
"net/http"
"testing"
"time"
)
// withStubbedPerpDexFetch swaps the live fetch function and resets the cache,
// restoring both when the test finishes.
func withStubbedPerpDexFetch(t *testing.T, fn func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error)) {
t.Helper()
original := fetchPerpDexCoinsFn
fetchPerpDexCoinsFn = fn
perpDexCoinCache.mu.Lock()
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
perpDexCoinCache.mu.Unlock()
t.Cleanup(func() {
fetchPerpDexCoinsFn = original
perpDexCoinCache.mu.Lock()
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
perpDexCoinCache.mu.Unlock()
})
}
func TestGetPerpDexCoinsCachesWithinTTL(t *testing.T) {
calls := 0
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
calls++
return []CoinInfo{{Symbol: "xyz:TSLA", MarkPrice: 400}}, nil
})
first, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("first call: %v", err)
}
second, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("second call: %v", err)
}
if calls != 1 {
t.Fatalf("fetch calls = %d, want 1 (second call must hit cache)", calls)
}
if len(first) != 1 || len(second) != 1 || second[0].Symbol != "xyz:TSLA" {
t.Fatalf("unexpected results: first=%v second=%v", first, second)
}
}
func TestGetPerpDexCoinsServesStaleOnUpstreamError(t *testing.T) {
calls := 0
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
calls++
if calls == 1 {
return []CoinInfo{{Symbol: "xyz:NVDA", MarkPrice: 1000}}, nil
}
return nil, errors.New("API returned status 429")
})
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err != nil {
t.Fatalf("first call: %v", err)
}
// Expire the cache so the next call must attempt a refresh.
perpDexCoinCache.mu.Lock()
entry := perpDexCoinCache.entries["xyz"]
entry.fetchedAt = time.Now().Add(-2 * perpDexCacheTTL)
perpDexCoinCache.entries["xyz"] = entry
perpDexCoinCache.mu.Unlock()
coins, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("expected stale data instead of error, got: %v", err)
}
if len(coins) != 1 || coins[0].Symbol != "xyz:NVDA" {
t.Fatalf("expected stale NVDA entry, got %v", coins)
}
if calls != 2 {
t.Fatalf("fetch calls = %d, want 2 (refresh attempted)", calls)
}
}
func TestGetPerpDexCoinsErrorsWithoutAnyCache(t *testing.T) {
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
return nil, errors.New("API returned status 429")
})
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err == nil {
t.Fatal("expected error when upstream fails and no cache exists")
}
}
func TestGetPerpDexCoinsCachesPerDex(t *testing.T) {
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
if dex == "xyz" {
return []CoinInfo{{Symbol: "xyz:AAPL"}}, nil
}
return []CoinInfo{{Symbol: "BTC"}}, nil
})
xyz, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("xyz: %v", err)
}
def, err := GetPerpDexCoins(context.Background(), "")
if err != nil {
t.Fatalf("default dex: %v", err)
}
if xyz[0].Symbol != "xyz:AAPL" || def[0].Symbol != "BTC" {
t.Fatalf("cache keys collided: xyz=%v default=%v", xyz, def)
}
}

View File

@@ -5,6 +5,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -180,14 +181,8 @@ func deriveAsterOrderAction(side, positionSide string, realizedPnL float64) stri
} }
// StartOrderSync starts background order sync task for Aster // StartOrderSync starts background order sync task for Aster
func (t *AsterTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *AsterTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Aster", func() error {
go func() { return t.SyncOrdersFromAster(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromAster(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Aster order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Aster order sync started (interval: %v)", interval)
} }

View File

@@ -421,7 +421,7 @@ func (at *AutoTrader) Run() error {
// Start Lighter order sync if using Lighter exchange // Start Lighter order sync if using Lighter exchange
if at.exchange == "lighter" { if at.exchange == "lighter" {
if lighterTrader, ok := at.trader.(*lighter.LighterTraderV2); ok && at.store != nil { if lighterTrader, ok := at.trader.(*lighter.LighterTraderV2); ok && at.store != nil {
lighterTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) lighterTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Lighter order+position sync enabled (every 30s)") at.logInfof("🔄 Lighter order+position sync enabled (every 30s)")
} }
} }
@@ -429,7 +429,7 @@ func (at *AutoTrader) Run() error {
// Start Hyperliquid order sync if using Hyperliquid exchange // Start Hyperliquid order sync if using Hyperliquid exchange
if at.exchange == "hyperliquid" { if at.exchange == "hyperliquid" {
if hyperliquidTrader, ok := at.trader.(*hyperliquid.HyperliquidTrader); ok && at.store != nil { if hyperliquidTrader, ok := at.trader.(*hyperliquid.HyperliquidTrader); ok && at.store != nil {
hyperliquidTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) hyperliquidTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Hyperliquid order+position sync enabled (every 30s)") at.logInfof("🔄 Hyperliquid order+position sync enabled (every 30s)")
} }
} }
@@ -437,7 +437,7 @@ func (at *AutoTrader) Run() error {
// Start Bybit order sync if using Bybit exchange // Start Bybit order sync if using Bybit exchange
if at.exchange == "bybit" { if at.exchange == "bybit" {
if bybitTrader, ok := at.trader.(*bybit.BybitTrader); ok && at.store != nil { if bybitTrader, ok := at.trader.(*bybit.BybitTrader); ok && at.store != nil {
bybitTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) bybitTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Bybit order+position sync enabled (every 30s)") at.logInfof("🔄 Bybit order+position sync enabled (every 30s)")
} }
} }
@@ -445,7 +445,7 @@ func (at *AutoTrader) Run() error {
// Start OKX order sync if using OKX exchange // Start OKX order sync if using OKX exchange
if at.exchange == "okx" { if at.exchange == "okx" {
if okxTrader, ok := at.trader.(*okx.OKXTrader); ok && at.store != nil { if okxTrader, ok := at.trader.(*okx.OKXTrader); ok && at.store != nil {
okxTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) okxTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 OKX order+position sync enabled (every 30s)") at.logInfof("🔄 OKX order+position sync enabled (every 30s)")
} }
} }
@@ -453,7 +453,7 @@ func (at *AutoTrader) Run() error {
// Start Bitget order sync if using Bitget exchange // Start Bitget order sync if using Bitget exchange
if at.exchange == "bitget" { if at.exchange == "bitget" {
if bitgetTrader, ok := at.trader.(*bitget.BitgetTrader); ok && at.store != nil { if bitgetTrader, ok := at.trader.(*bitget.BitgetTrader); ok && at.store != nil {
bitgetTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) bitgetTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Bitget order+position sync enabled (every 30s)") at.logInfof("🔄 Bitget order+position sync enabled (every 30s)")
} }
} }
@@ -461,7 +461,7 @@ func (at *AutoTrader) Run() error {
// Start Aster order sync if using Aster exchange // Start Aster order sync if using Aster exchange
if at.exchange == "aster" { if at.exchange == "aster" {
if asterTrader, ok := at.trader.(*aster.AsterTrader); ok && at.store != nil { if asterTrader, ok := at.trader.(*aster.AsterTrader); ok && at.store != nil {
asterTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) asterTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Aster order+position sync enabled (every 30s)") at.logInfof("🔄 Aster order+position sync enabled (every 30s)")
} }
} }
@@ -469,7 +469,7 @@ func (at *AutoTrader) Run() error {
// Start Binance order sync if using Binance exchange // Start Binance order sync if using Binance exchange
if at.exchange == "binance" { if at.exchange == "binance" {
if binanceTrader, ok := at.trader.(*binance.FuturesTrader); ok && at.store != nil { if binanceTrader, ok := at.trader.(*binance.FuturesTrader); ok && at.store != nil {
binanceTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) binanceTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Binance order+position sync enabled (every 30s)") at.logInfof("🔄 Binance order+position sync enabled (every 30s)")
} }
} }
@@ -477,7 +477,7 @@ func (at *AutoTrader) Run() error {
// Start Gate order sync if using Gate exchange // Start Gate order sync if using Gate exchange
if at.exchange == "gate" { if at.exchange == "gate" {
if gateTrader, ok := at.trader.(*gate.GateTrader); ok && at.store != nil { if gateTrader, ok := at.trader.(*gate.GateTrader); ok && at.store != nil {
gateTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) gateTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 Gate order+position sync enabled (every 30s)") at.logInfof("🔄 Gate order+position sync enabled (every 30s)")
} }
} }
@@ -485,7 +485,7 @@ func (at *AutoTrader) Run() error {
// Start KuCoin order sync if using KuCoin exchange // Start KuCoin order sync if using KuCoin exchange
if at.exchange == "kucoin" { if at.exchange == "kucoin" {
if kucoinTrader, ok := at.trader.(*kucoin.KuCoinTrader); ok && at.store != nil { if kucoinTrader, ok := at.trader.(*kucoin.KuCoinTrader); ok && at.store != nil {
kucoinTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second) kucoinTrader.StartOrderSync(at.id, at.exchangeID, at.exchange, at.store, 30*time.Second, at.stopMonitorCh)
at.logInfof("🔄 KuCoin order+position sync enabled (every 30s)") at.logInfof("🔄 KuCoin order+position sync enabled (every 30s)")
} }
} }

View File

@@ -5,6 +5,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"nofx/trader/types" "nofx/trader/types"
"sort" "sort"
"strings" "strings"
@@ -349,7 +350,7 @@ func (t *FuturesTrader) determineOrderAction(side, positionSide string, realized
} }
// StartOrderSync starts background order sync task for Binance // StartOrderSync starts background order sync task for Binance
func (t *FuturesTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *FuturesTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
// Run first sync immediately // Run first sync immediately
go func() { go func() {
logger.Infof("🔄 Running initial Binance order sync...") logger.Infof("🔄 Running initial Binance order sync...")
@@ -359,13 +360,7 @@ func (t *FuturesTrader) StartOrderSync(traderID string, exchangeID string, excha
}() }()
// Then run periodically // Then run periodically
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Binance", func() error {
go func() { return t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromBinance(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Binance order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Binance order sync started (interval: %v)", interval)
} }

View File

@@ -6,6 +6,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -48,7 +49,6 @@ func (t *BitgetTrader) GetTrades(startTime time.Time, limit int) ([]BitgetTrade,
return nil, fmt.Errorf("failed to get fill history: %w", err) return nil, fmt.Errorf("failed to get fill history: %w", err)
} }
// Bitget fill structure - supports both one-way and hedge mode // Bitget fill structure - supports both one-way and hedge mode
type BitgetFill struct { type BitgetFill struct {
TradeID string `json:"tradeId"` TradeID string `json:"tradeId"`
@@ -279,14 +279,8 @@ func (t *BitgetTrader) SyncOrdersFromBitget(traderID string, exchangeID string,
} }
// StartOrderSync starts background order sync task for Bitget // StartOrderSync starts background order sync task for Bitget
func (t *BitgetTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *BitgetTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Bitget", func() error {
go func() { return t.SyncOrdersFromBitget(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromBitget(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Bitget order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Bitget order sync started (interval: %v)", interval)
} }

View File

@@ -11,6 +11,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -298,14 +299,8 @@ func (t *BybitTrader) SyncOrdersFromBybit(traderID string, exchangeID string, ex
} }
// StartOrderSync starts background order sync task for Bybit // StartOrderSync starts background order sync task for Bybit
func (t *BybitTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *BybitTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Bybit", func() error {
go func() { return t.SyncOrdersFromBybit(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromBybit(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Bybit order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Bybit order sync started (interval: %v)", interval)
} }

View File

@@ -5,6 +5,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -291,14 +292,8 @@ func (t *GateTrader) SyncOrdersFromGate(traderID string, exchangeID string, exch
} }
// StartOrderSync starts background order sync task for Gate // StartOrderSync starts background order sync task for Gate
func (t *GateTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *GateTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Gate", func() error {
go func() { return t.SyncOrdersFromGate(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromGate(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Gate order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Gate order sync started (interval: %v)", interval)
} }

View File

@@ -5,6 +5,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -44,87 +45,87 @@ func (t *HyperliquidTrader) SyncOrdersFromHyperliquid(traderID string, exchangeI
syncedCount := 0 syncedCount := 0
for _, trade := range trades { for _, trade := range trades {
// Check if trade already exists (use exchangeID which is UUID, not exchange type) // Check if trade already exists (use exchangeID which is UUID, not exchange type)
existing, err := orderStore.GetOrderByExchangeID(exchangeID, trade.TradeID) existing, err := orderStore.GetOrderByExchangeID(exchangeID, trade.TradeID)
if err == nil && existing != nil { if err == nil && existing != nil {
continue // Order already exists, skip continue // Order already exists, skip
} }
// Normalize symbol // Normalize symbol
symbol := market.Normalize(trade.Symbol) symbol := market.Normalize(trade.Symbol)
// Use order action from trade (parsed from Hyperliquid Dir field) // Use order action from trade (parsed from Hyperliquid Dir field)
// Dir field values: "Open Long", "Open Short", "Close Long", "Close Short" // Dir field values: "Open Long", "Open Short", "Close Long", "Close Short"
orderAction := trade.OrderAction orderAction := trade.OrderAction
positionSide := "LONG" positionSide := "LONG"
if strings.Contains(orderAction, "short") { if strings.Contains(orderAction, "short") {
positionSide = "SHORT" positionSide = "SHORT"
} }
// Create order record - use Unix milliseconds UTC // Create order record - use Unix milliseconds UTC
tradeTimeMs := trade.Time.UTC().UnixMilli() tradeTimeMs := trade.Time.UTC().UnixMilli()
orderRecord := &store.TraderOrder{ orderRecord := &store.TraderOrder{
TraderID: traderID, TraderID: traderID,
ExchangeID: exchangeID, // UUID ExchangeID: exchangeID, // UUID
ExchangeType: exchangeType, // Exchange type ExchangeType: exchangeType, // Exchange type
ExchangeOrderID: trade.TradeID, ExchangeOrderID: trade.TradeID,
Symbol: symbol, Symbol: symbol,
Side: trade.Side, Side: trade.Side,
PositionSide: "BOTH", // Hyperliquid uses one-way position mode PositionSide: "BOTH", // Hyperliquid uses one-way position mode
Type: "MARKET", Type: "MARKET",
OrderAction: orderAction, OrderAction: orderAction,
Quantity: trade.Quantity, Quantity: trade.Quantity,
Price: trade.Price, Price: trade.Price,
Status: "FILLED", Status: "FILLED",
FilledQuantity: trade.Quantity, FilledQuantity: trade.Quantity,
AvgFillPrice: trade.Price, AvgFillPrice: trade.Price,
Commission: trade.Fee, Commission: trade.Fee,
FilledAt: tradeTimeMs, FilledAt: tradeTimeMs,
CreatedAt: tradeTimeMs, CreatedAt: tradeTimeMs,
UpdatedAt: tradeTimeMs, UpdatedAt: tradeTimeMs,
} }
// Insert order record // Insert order record
if err := orderStore.CreateOrder(orderRecord); err != nil { if err := orderStore.CreateOrder(orderRecord); err != nil {
logger.Infof(" ⚠️ Failed to sync trade %s: %v", trade.TradeID, err) logger.Infof(" ⚠️ Failed to sync trade %s: %v", trade.TradeID, err)
continue continue
} }
// Create fill record - use Unix milliseconds UTC // Create fill record - use Unix milliseconds UTC
fillRecord := &store.TraderFill{ fillRecord := &store.TraderFill{
TraderID: traderID, TraderID: traderID,
ExchangeID: exchangeID, // UUID ExchangeID: exchangeID, // UUID
ExchangeType: exchangeType, // Exchange type ExchangeType: exchangeType, // Exchange type
OrderID: orderRecord.ID, OrderID: orderRecord.ID,
ExchangeOrderID: trade.TradeID, ExchangeOrderID: trade.TradeID,
ExchangeTradeID: trade.TradeID, ExchangeTradeID: trade.TradeID,
Symbol: symbol, Symbol: symbol,
Side: trade.Side, Side: trade.Side,
Price: trade.Price, Price: trade.Price,
Quantity: trade.Quantity, Quantity: trade.Quantity,
QuoteQuantity: trade.Price * trade.Quantity, QuoteQuantity: trade.Price * trade.Quantity,
Commission: trade.Fee, Commission: trade.Fee,
CommissionAsset: "USDT", CommissionAsset: "USDT",
RealizedPnL: trade.RealizedPnL, RealizedPnL: trade.RealizedPnL,
IsMaker: false, // Hyperliquid GetTrades doesn't provide maker/taker info IsMaker: false, // Hyperliquid GetTrades doesn't provide maker/taker info
CreatedAt: tradeTimeMs, CreatedAt: tradeTimeMs,
} }
if err := orderStore.CreateFill(fillRecord); err != nil { if err := orderStore.CreateFill(fillRecord); err != nil {
logger.Infof(" ⚠️ Failed to sync fill for trade %s: %v", trade.TradeID, err) logger.Infof(" ⚠️ Failed to sync fill for trade %s: %v", trade.TradeID, err)
} }
// Create/update position record using PositionBuilder // Create/update position record using PositionBuilder
if err := posBuilder.ProcessTrade( if err := posBuilder.ProcessTrade(
traderID, exchangeID, exchangeType, traderID, exchangeID, exchangeType,
symbol, positionSide, orderAction, symbol, positionSide, orderAction,
trade.Quantity, trade.Price, trade.Fee, trade.RealizedPnL, trade.Quantity, trade.Price, trade.Fee, trade.RealizedPnL,
tradeTimeMs, trade.TradeID, tradeTimeMs, trade.TradeID,
); err != nil { ); err != nil {
logger.Infof(" ⚠️ Failed to sync position for trade %s: %v", trade.TradeID, err) logger.Infof(" ⚠️ Failed to sync position for trade %s: %v", trade.TradeID, err)
} else { } else {
logger.Infof(" 📍 Position updated for trade: %s (action: %s, qty: %.6f)", trade.TradeID, orderAction, trade.Quantity) logger.Infof(" 📍 Position updated for trade: %s (action: %s, qty: %.6f)", trade.TradeID, orderAction, trade.Quantity)
} }
syncedCount++ syncedCount++
logger.Infof(" ✅ Synced trade: %s %s %s qty=%.6f price=%.6f pnl=%.2f fee=%.6f action=%s", logger.Infof(" ✅ Synced trade: %s %s %s qty=%.6f price=%.6f pnl=%.2f fee=%.6f action=%s",
@@ -136,14 +137,8 @@ func (t *HyperliquidTrader) SyncOrdersFromHyperliquid(traderID string, exchangeI
} }
// StartOrderSync starts background order sync task // StartOrderSync starts background order sync task
func (t *HyperliquidTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *HyperliquidTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Hyperliquid", func() error {
go func() { return t.SyncOrdersFromHyperliquid(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromHyperliquid(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ Hyperliquid order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 Hyperliquid order sync started (interval: %v)", interval)
} }

View File

@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"nofx/logger" "nofx/logger"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"nofx/trader/types" "nofx/trader/types"
"sort" "sort"
"strings" "strings"
@@ -399,14 +400,8 @@ func (t *KuCoinTrader) SyncOrdersFromKuCoin(traderID string, exchangeID string,
} }
// StartOrderSync starts background order sync task for KuCoin // StartOrderSync starts background order sync task for KuCoin
func (t *KuCoinTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *KuCoinTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "KuCoin", func() error {
go func() { return t.SyncOrdersFromKuCoin(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromKuCoin(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ KuCoin order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 KuCoin order sync started (interval: %v)", interval)
} }

View File

@@ -5,6 +5,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -145,17 +146,14 @@ func (t *LighterTraderV2) SyncOrdersFromLighter(traderID string, exchangeID stri
} }
// StartOrderSync starts background order sync task // StartOrderSync starts background order sync task
func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *LighterTraderV2) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "Lighter", func() error {
go func() { err := t.SyncOrdersFromLighter(traderID, exchangeID, exchangeType, st)
for range ticker.C { // A 404 just means the account has no fills yet — treat as a
if err := t.SyncOrdersFromLighter(traderID, exchangeID, exchangeType, st); err != nil { // successful empty sync to avoid log spam and pointless backoff.
// Only log non-404 errors to reduce log spam if err != nil && strings.Contains(err.Error(), "status 404") {
if !strings.Contains(err.Error(), "status 404") { return nil
logger.Infof("⚠️ Order sync failed: %v", err)
}
}
} }
}() return err
logger.Infof("🔄 Lighter order+position sync started (interval: %v)", interval) })
} }

View File

@@ -6,6 +6,7 @@ import (
"nofx/logger" "nofx/logger"
"nofx/market" "nofx/market"
"nofx/store" "nofx/store"
"nofx/trader/syncloop"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -272,14 +273,8 @@ func (t *OKXTrader) SyncOrdersFromOKX(traderID string, exchangeID string, exchan
} }
// StartOrderSync starts background order sync task for OKX // StartOrderSync starts background order sync task for OKX
func (t *OKXTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration) { func (t *OKXTrader) StartOrderSync(traderID string, exchangeID string, exchangeType string, st *store.Store, interval time.Duration, stop <-chan struct{}) {
ticker := time.NewTicker(interval) syncloop.Run(stop, interval, "OKX", func() error {
go func() { return t.SyncOrdersFromOKX(traderID, exchangeID, exchangeType, st)
for range ticker.C { })
if err := t.SyncOrdersFromOKX(traderID, exchangeID, exchangeType, st); err != nil {
logger.Infof("⚠️ OKX order sync failed: %v", err)
}
}
}()
logger.Infof("🔄 OKX order sync started (interval: %v)", interval)
} }

View File

@@ -0,0 +1,48 @@
// Package syncloop runs background exchange order-sync loops with a shared
// lifecycle: loops stop when the owning trader stops, and consecutive
// failures back off exponentially so a rate-limiting exchange (HTTP 429)
// is not hammered at full frequency.
package syncloop
import (
"time"
"nofx/logger"
)
// maxBackoff caps the failure backoff so a recovered exchange is picked up
// within a few minutes at worst.
const maxBackoff = 5 * time.Minute
// Run starts a background loop calling syncFn at the given interval until
// stop is closed. After each consecutive failure the wait doubles (capped at
// maxBackoff); the first success resets it to the base interval.
func Run(stop <-chan struct{}, interval time.Duration, name string, syncFn func() error) {
if interval <= 0 {
interval = 30 * time.Second
}
go func() {
wait := interval
timer := time.NewTimer(wait)
defer timer.Stop()
for {
select {
case <-stop:
logger.Infof("⏹ %s order sync stopped", name)
return
case <-timer.C:
if err := syncFn(); err != nil {
wait *= 2
if wait > maxBackoff {
wait = maxBackoff
}
logger.Infof("⚠️ %s order sync failed: %v (backing off, next attempt in %v)", name, err, wait)
} else {
wait = interval
}
timer.Reset(wait)
}
}
}()
logger.Infof("🔄 %s order sync started (interval: %v)", name, interval)
}

View File

@@ -0,0 +1,78 @@
package syncloop
import (
"errors"
"sync/atomic"
"testing"
"time"
)
func TestRunStopsWhenStopChannelCloses(t *testing.T) {
stop := make(chan struct{})
var calls atomic.Int64
Run(stop, 5*time.Millisecond, "test", func() error {
calls.Add(1)
return nil
})
// Let a few ticks happen, then stop.
time.Sleep(30 * time.Millisecond)
close(stop)
time.Sleep(20 * time.Millisecond)
after := calls.Load()
if after == 0 {
t.Fatal("sync function never ran")
}
// No further calls after stop.
time.Sleep(40 * time.Millisecond)
if calls.Load() != after {
t.Fatalf("sync kept running after stop: %d -> %d", after, calls.Load())
}
}
func TestRunBacksOffOnConsecutiveFailures(t *testing.T) {
stop := make(chan struct{})
defer close(stop)
var calls atomic.Int64
Run(stop, 10*time.Millisecond, "test", func() error {
calls.Add(1)
return errors.New("API returned status 429")
})
// With a 10ms base interval and exponential backoff (10, 20, 40, 80...),
// 100ms allows at most ~4 failing attempts. Without backoff there would
// be ~10.
time.Sleep(100 * time.Millisecond)
got := calls.Load()
if got == 0 {
t.Fatal("sync function never ran")
}
if got > 5 {
t.Fatalf("expected backoff to throttle failing sync, got %d calls in 100ms", got)
}
}
func TestRunRecoversIntervalAfterSuccess(t *testing.T) {
stop := make(chan struct{})
defer close(stop)
var calls atomic.Int64
// Fail twice, then succeed forever.
Run(stop, 5*time.Millisecond, "test", func() error {
n := calls.Add(1)
if n <= 2 {
return errors.New("transient")
}
return nil
})
// Failures at 5ms and 15ms (backoff 10ms), success at ~35ms (backoff 20ms),
// then the interval resets to 5ms — plenty of successful runs by 150ms.
time.Sleep(150 * time.Millisecond)
if got := calls.Load(); got < 8 {
t.Fatalf("expected interval to reset after success, got only %d calls", got)
}
}