fix(trader): stop order-sync goroutine leak and rate-limit hammering

Every StartOrderSync spawned a ticker goroutine that ran forever — it
survived trader stop AND deletion, so each quick-created trader left a
permanent 30s Hyperliquid poll behind. Stacked leaks turned into an
~8s effective hammer that tripped Hyperliquid's 429 rate limit, which
then broke the symbol board, trader creation, and order sync itself.

- new trader/syncloop package: shared stoppable sync loop with
  exponential failure backoff (30s base, 5min cap)
- all 9 exchanges' StartOrderSync now take the trader's stop channel
  and stop when the trader stops (close broadcast from AutoTrader.Stop)
- provider/hyperliquid: GetPerpDexCoins now serves a 5min TTL cache and
  falls back to the stale board when the upstream returns 429, so the
  symbol panel keeps working through rate limiting
This commit is contained in:
tinkle-community
2026-06-11 21:45:31 +08:00
parent 133ef51de8
commit 953240565f
14 changed files with 424 additions and 178 deletions

View File

@@ -164,9 +164,59 @@ func fetchPerpDexCoins(ctx context.Context, client *http.Client, dex string) ([]
return coins, nil
}
// GetPerpDexCoins fetches current tradable USDC perp assets for a given Hyperliquid dex.
// perpDexCacheTTL bounds how often the perp-dex symbol board is re-fetched.
// The tradable symbol list changes rarely; prices/volume on the board are
// display hints, so short staleness is far better than hammering the
// Hyperliquid API (which rate-limits with 429) on every panel render.
const perpDexCacheTTL = 5 * time.Minute
type perpDexCacheEntry struct {
coins []CoinInfo
fetchedAt time.Time
}
type perpDexCacheStore struct {
mu sync.Mutex
entries map[string]perpDexCacheEntry
}
var perpDexCoinCache = &perpDexCacheStore{entries: map[string]perpDexCacheEntry{}}
// fetchPerpDexCoinsFn is swappable in tests.
var fetchPerpDexCoinsFn = fetchPerpDexCoins
// GetPerpDexCoins returns current tradable USDC perp assets for a given
// Hyperliquid dex, served from a TTL cache. When the upstream fetch fails
// (e.g. HTTP 429 rate limiting) and stale data exists, the stale board is
// served instead of an error so the UI keeps working.
func GetPerpDexCoins(ctx context.Context, dex string) ([]CoinInfo, error) {
return fetchPerpDexCoins(ctx, &http.Client{Timeout: 30 * time.Second}, dex)
perpDexCoinCache.mu.Lock()
defer perpDexCoinCache.mu.Unlock()
entry, hasCache := perpDexCoinCache.entries[dex]
if hasCache && time.Since(entry.fetchedAt) < perpDexCacheTTL {
return copyCoins(entry.coins), nil
}
coins, err := fetchPerpDexCoinsFn(ctx, &http.Client{Timeout: 30 * time.Second}, dex)
if err != nil {
if hasCache {
logger.Infof("⚠️ Hyperliquid perp-dex fetch failed (%v); serving cached board for dex %q from %s",
err, dex, entry.fetchedAt.Format(time.RFC3339))
return copyCoins(entry.coins), nil
}
return nil, err
}
perpDexCoinCache.entries[dex] = perpDexCacheEntry{coins: coins, fetchedAt: time.Now()}
return copyCoins(coins), nil
}
// copyCoins returns a defensive copy so callers cannot mutate the cache.
func copyCoins(coins []CoinInfo) []CoinInfo {
out := make([]CoinInfo, len(coins))
copy(out, coins)
return out
}
// fetchCoins fetches all default Hyperliquid crypto coins and sorts by volume

View File

@@ -0,0 +1,113 @@
package hyperliquid
import (
"context"
"errors"
"net/http"
"testing"
"time"
)
// withStubbedPerpDexFetch swaps the live fetch function and resets the cache,
// restoring both when the test finishes.
func withStubbedPerpDexFetch(t *testing.T, fn func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error)) {
t.Helper()
original := fetchPerpDexCoinsFn
fetchPerpDexCoinsFn = fn
perpDexCoinCache.mu.Lock()
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
perpDexCoinCache.mu.Unlock()
t.Cleanup(func() {
fetchPerpDexCoinsFn = original
perpDexCoinCache.mu.Lock()
perpDexCoinCache.entries = map[string]perpDexCacheEntry{}
perpDexCoinCache.mu.Unlock()
})
}
func TestGetPerpDexCoinsCachesWithinTTL(t *testing.T) {
calls := 0
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
calls++
return []CoinInfo{{Symbol: "xyz:TSLA", MarkPrice: 400}}, nil
})
first, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("first call: %v", err)
}
second, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("second call: %v", err)
}
if calls != 1 {
t.Fatalf("fetch calls = %d, want 1 (second call must hit cache)", calls)
}
if len(first) != 1 || len(second) != 1 || second[0].Symbol != "xyz:TSLA" {
t.Fatalf("unexpected results: first=%v second=%v", first, second)
}
}
func TestGetPerpDexCoinsServesStaleOnUpstreamError(t *testing.T) {
calls := 0
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
calls++
if calls == 1 {
return []CoinInfo{{Symbol: "xyz:NVDA", MarkPrice: 1000}}, nil
}
return nil, errors.New("API returned status 429")
})
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err != nil {
t.Fatalf("first call: %v", err)
}
// Expire the cache so the next call must attempt a refresh.
perpDexCoinCache.mu.Lock()
entry := perpDexCoinCache.entries["xyz"]
entry.fetchedAt = time.Now().Add(-2 * perpDexCacheTTL)
perpDexCoinCache.entries["xyz"] = entry
perpDexCoinCache.mu.Unlock()
coins, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("expected stale data instead of error, got: %v", err)
}
if len(coins) != 1 || coins[0].Symbol != "xyz:NVDA" {
t.Fatalf("expected stale NVDA entry, got %v", coins)
}
if calls != 2 {
t.Fatalf("fetch calls = %d, want 2 (refresh attempted)", calls)
}
}
func TestGetPerpDexCoinsErrorsWithoutAnyCache(t *testing.T) {
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
return nil, errors.New("API returned status 429")
})
if _, err := GetPerpDexCoins(context.Background(), "xyz"); err == nil {
t.Fatal("expected error when upstream fails and no cache exists")
}
}
func TestGetPerpDexCoinsCachesPerDex(t *testing.T) {
withStubbedPerpDexFetch(t, func(ctx context.Context, client *http.Client, dex string) ([]CoinInfo, error) {
if dex == "xyz" {
return []CoinInfo{{Symbol: "xyz:AAPL"}}, nil
}
return []CoinInfo{{Symbol: "BTC"}}, nil
})
xyz, err := GetPerpDexCoins(context.Background(), "xyz")
if err != nil {
t.Fatalf("xyz: %v", err)
}
def, err := GetPerpDexCoins(context.Background(), "")
if err != nil {
t.Fatalf("default dex: %v", err)
}
if xyz[0].Symbol != "xyz:AAPL" || def[0].Symbol != "BTC" {
t.Fatalf("cache keys collided: xyz=%v default=%v", xyz, def)
}
}