fix(trader): stop order-sync goroutine leak and rate-limit hammering

Every StartOrderSync spawned a ticker goroutine that ran forever — it
survived trader stop AND deletion, so each quick-created trader left a
permanent 30s Hyperliquid poll behind. Stacked leaks turned into an
~8s effective hammer that tripped Hyperliquid's 429 rate limit, which
then broke the symbol board, trader creation, and order sync itself.

- new trader/syncloop package: shared stoppable sync loop with
  exponential failure backoff (30s base, 5min cap)
- all 9 exchanges' StartOrderSync now take the trader's stop channel
  and stop when the trader stops (close broadcast from AutoTrader.Stop)
- provider/hyperliquid: GetPerpDexCoins now serves a 5min TTL cache and
  falls back to the stale board when the upstream returns 429, so the
  symbol panel keeps working through rate limiting
This commit is contained in:
tinkle-community
2026-06-11 21:45:31 +08:00
parent 133ef51de8
commit 953240565f
14 changed files with 424 additions and 178 deletions

View File

@@ -0,0 +1,48 @@
// Package syncloop runs background exchange order-sync loops with a shared
// lifecycle: loops stop when the owning trader stops, and consecutive
// failures back off exponentially so a rate-limiting exchange (HTTP 429)
// is not hammered at full frequency.
package syncloop
import (
"time"
"nofx/logger"
)
// maxBackoff caps the failure backoff so a recovered exchange is picked up
// within a few minutes at worst.
const maxBackoff = 5 * time.Minute
// Run starts a background loop calling syncFn at the given interval until
// stop is closed. After each consecutive failure the wait doubles (capped at
// maxBackoff); the first success resets it to the base interval.
func Run(stop <-chan struct{}, interval time.Duration, name string, syncFn func() error) {
if interval <= 0 {
interval = 30 * time.Second
}
go func() {
wait := interval
timer := time.NewTimer(wait)
defer timer.Stop()
for {
select {
case <-stop:
logger.Infof("⏹ %s order sync stopped", name)
return
case <-timer.C:
if err := syncFn(); err != nil {
wait *= 2
if wait > maxBackoff {
wait = maxBackoff
}
logger.Infof("⚠️ %s order sync failed: %v (backing off, next attempt in %v)", name, err, wait)
} else {
wait = interval
}
timer.Reset(wait)
}
}
}()
logger.Infof("🔄 %s order sync started (interval: %v)", name, interval)
}

View File

@@ -0,0 +1,78 @@
package syncloop
import (
"errors"
"sync/atomic"
"testing"
"time"
)
func TestRunStopsWhenStopChannelCloses(t *testing.T) {
stop := make(chan struct{})
var calls atomic.Int64
Run(stop, 5*time.Millisecond, "test", func() error {
calls.Add(1)
return nil
})
// Let a few ticks happen, then stop.
time.Sleep(30 * time.Millisecond)
close(stop)
time.Sleep(20 * time.Millisecond)
after := calls.Load()
if after == 0 {
t.Fatal("sync function never ran")
}
// No further calls after stop.
time.Sleep(40 * time.Millisecond)
if calls.Load() != after {
t.Fatalf("sync kept running after stop: %d -> %d", after, calls.Load())
}
}
func TestRunBacksOffOnConsecutiveFailures(t *testing.T) {
stop := make(chan struct{})
defer close(stop)
var calls atomic.Int64
Run(stop, 10*time.Millisecond, "test", func() error {
calls.Add(1)
return errors.New("API returned status 429")
})
// With a 10ms base interval and exponential backoff (10, 20, 40, 80...),
// 100ms allows at most ~4 failing attempts. Without backoff there would
// be ~10.
time.Sleep(100 * time.Millisecond)
got := calls.Load()
if got == 0 {
t.Fatal("sync function never ran")
}
if got > 5 {
t.Fatalf("expected backoff to throttle failing sync, got %d calls in 100ms", got)
}
}
func TestRunRecoversIntervalAfterSuccess(t *testing.T) {
stop := make(chan struct{})
defer close(stop)
var calls atomic.Int64
// Fail twice, then succeed forever.
Run(stop, 5*time.Millisecond, "test", func() error {
n := calls.Add(1)
if n <= 2 {
return errors.New("transient")
}
return nil
})
// Failures at 5ms and 15ms (backoff 10ms), success at ~35ms (backoff 20ms),
// then the interval resets to 5ms — plenty of successful runs by 150ms.
time.Sleep(150 * time.Millisecond)
if got := calls.Load(); got < 8 {
t.Fatalf("expected interval to reset after success, got only %d calls", got)
}
}