mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-14 08:16:56 +08:00
fix(trader): stop order-sync goroutine leak and rate-limit hammering
Every StartOrderSync spawned a ticker goroutine that ran forever — it survived trader stop AND deletion, so each quick-created trader left a permanent 30s Hyperliquid poll behind. Stacked leaks turned into an ~8s effective hammer that tripped Hyperliquid's 429 rate limit, which then broke the symbol board, trader creation, and order sync itself. - new trader/syncloop package: shared stoppable sync loop with exponential failure backoff (30s base, 5min cap) - all 9 exchanges' StartOrderSync now take the trader's stop channel and stop when the trader stops (close broadcast from AutoTrader.Stop) - provider/hyperliquid: GetPerpDexCoins now serves a 5min TTL cache and falls back to the stale board when the upstream returns 429, so the symbol panel keeps working through rate limiting
This commit is contained in:
@@ -164,9 +164,59 @@ func fetchPerpDexCoins(ctx context.Context, client *http.Client, dex string) ([]
|
||||
return coins, nil
|
||||
}
|
||||
|
||||
// GetPerpDexCoins fetches current tradable USDC perp assets for a given Hyperliquid dex.
|
||||
// perpDexCacheTTL bounds how often the perp-dex symbol board is re-fetched.
|
||||
// The tradable symbol list changes rarely; prices/volume on the board are
|
||||
// display hints, so short staleness is far better than hammering the
|
||||
// Hyperliquid API (which rate-limits with 429) on every panel render.
|
||||
const perpDexCacheTTL = 5 * time.Minute
|
||||
|
||||
type perpDexCacheEntry struct {
|
||||
coins []CoinInfo
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
type perpDexCacheStore struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]perpDexCacheEntry
|
||||
}
|
||||
|
||||
var perpDexCoinCache = &perpDexCacheStore{entries: map[string]perpDexCacheEntry{}}
|
||||
|
||||
// fetchPerpDexCoinsFn is swappable in tests.
|
||||
var fetchPerpDexCoinsFn = fetchPerpDexCoins
|
||||
|
||||
// GetPerpDexCoins returns current tradable USDC perp assets for a given
|
||||
// Hyperliquid dex, served from a TTL cache. When the upstream fetch fails
|
||||
// (e.g. HTTP 429 rate limiting) and stale data exists, the stale board is
|
||||
// served instead of an error so the UI keeps working.
|
||||
func GetPerpDexCoins(ctx context.Context, dex string) ([]CoinInfo, error) {
|
||||
return fetchPerpDexCoins(ctx, &http.Client{Timeout: 30 * time.Second}, dex)
|
||||
perpDexCoinCache.mu.Lock()
|
||||
defer perpDexCoinCache.mu.Unlock()
|
||||
|
||||
entry, hasCache := perpDexCoinCache.entries[dex]
|
||||
if hasCache && time.Since(entry.fetchedAt) < perpDexCacheTTL {
|
||||
return copyCoins(entry.coins), nil
|
||||
}
|
||||
|
||||
coins, err := fetchPerpDexCoinsFn(ctx, &http.Client{Timeout: 30 * time.Second}, dex)
|
||||
if err != nil {
|
||||
if hasCache {
|
||||
logger.Infof("⚠️ Hyperliquid perp-dex fetch failed (%v); serving cached board for dex %q from %s",
|
||||
err, dex, entry.fetchedAt.Format(time.RFC3339))
|
||||
return copyCoins(entry.coins), nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
perpDexCoinCache.entries[dex] = perpDexCacheEntry{coins: coins, fetchedAt: time.Now()}
|
||||
return copyCoins(coins), nil
|
||||
}
|
||||
|
||||
// copyCoins returns a defensive copy so callers cannot mutate the cache.
|
||||
func copyCoins(coins []CoinInfo) []CoinInfo {
|
||||
out := make([]CoinInfo, len(coins))
|
||||
copy(out, coins)
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchCoins fetches all default Hyperliquid crypto coins and sorts by volume
|
||||
|
||||
113
provider/hyperliquid/perp_dex_cache_test.go
Normal file
113
provider/hyperliquid/perp_dex_cache_test.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package hyperliquid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// withStubbedPerpDexFetch swaps the live fetch function and resets the cache,
|
||||
// restoring both when the test finishes.
|
||||
func withStubbedPerpDexFetch(t *testing.T, fn func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error)) {
|
||||
t.Helper()
|
||||
original := fetchPerpDexCoinsFn
|
||||
fetchPerpDexCoinsFn = fn
|
||||
perpDexCoinCache.mu.Lock()
|
||||
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
|
||||
perpDexCoinCache.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
fetchPerpDexCoinsFn = original
|
||||
perpDexCoinCache.mu.Lock()
|
||||
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
|
||||
perpDexCoinCache.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetPerpDexCoinsCachesWithinTTL(t *testing.T) {
|
||||
calls := 0
|
||||
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
|
||||
calls++
|
||||
return []CoinInfo{{Symbol: "xyz:TSLA", MarkPrice: 400}}, nil
|
||||
})
|
||||
|
||||
first, err := GetPerpDexCoins(context.Background(), "xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("first call: %v", err)
|
||||
}
|
||||
second, err := GetPerpDexCoins(context.Background(), "xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("second call: %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("fetch calls = %d, want 1 (second call must hit cache)", calls)
|
||||
}
|
||||
if len(first) != 1 || len(second) != 1 || second[0].Symbol != "xyz:TSLA" {
|
||||
t.Fatalf("unexpected results: first=%v second=%v", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPerpDexCoinsServesStaleOnUpstreamError(t *testing.T) {
|
||||
calls := 0
|
||||
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return []CoinInfo{{Symbol: "xyz:NVDA", MarkPrice: 1000}}, nil
|
||||
}
|
||||
return nil, errors.New("API returned status 429")
|
||||
})
|
||||
|
||||
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err != nil {
|
||||
t.Fatalf("first call: %v", err)
|
||||
}
|
||||
|
||||
// Expire the cache so the next call must attempt a refresh.
|
||||
perpDexCoinCache.mu.Lock()
|
||||
entry := perpDexCoinCache.entries["xyz"]
|
||||
entry.fetchedAt = time.Now().Add(-2 * perpDexCacheTTL)
|
||||
perpDexCoinCache.entries["xyz"] = entry
|
||||
perpDexCoinCache.mu.Unlock()
|
||||
|
||||
coins, err := GetPerpDexCoins(context.Background(), "xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("expected stale data instead of error, got: %v", err)
|
||||
}
|
||||
if len(coins) != 1 || coins[0].Symbol != "xyz:NVDA" {
|
||||
t.Fatalf("expected stale NVDA entry, got %v", coins)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("fetch calls = %d, want 2 (refresh attempted)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPerpDexCoinsErrorsWithoutAnyCache(t *testing.T) {
|
||||
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
|
||||
return nil, errors.New("API returned status 429")
|
||||
})
|
||||
|
||||
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err == nil {
|
||||
t.Fatal("expected error when upstream fails and no cache exists")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPerpDexCoinsCachesPerDex(t *testing.T) {
|
||||
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
|
||||
if dex == "xyz" {
|
||||
return []CoinInfo{{Symbol: "xyz:AAPL"}}, nil
|
||||
}
|
||||
return []CoinInfo{{Symbol: "BTC"}}, nil
|
||||
})
|
||||
|
||||
xyz, err := GetPerpDexCoins(context.Background(), "xyz")
|
||||
if err != nil {
|
||||
t.Fatalf("xyz: %v", err)
|
||||
}
|
||||
def, err := GetPerpDexCoins(context.Background(), "")
|
||||
if err != nil {
|
||||
t.Fatalf("default dex: %v", err)
|
||||
}
|
||||
if xyz[0].Symbol != "xyz:AAPL" || def[0].Symbol != "BTC" {
|
||||
t.Fatalf("cache keys collided: xyz=%v default=%v", xyz, def)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user