mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-15 00:36:56 +08:00
fix(trader): stop order-sync goroutine leak and rate-limit hammering
Every StartOrderSync spawned a ticker goroutine that ran forever — it survived trader stop AND deletion, so each quick-created trader left a permanent 30s Hyperliquid poll behind. Stacked leaks turned into an ~8s effective hammer that tripped Hyperliquid's 429 rate limit, which then broke the symbol board, trader creation, and order sync itself. - new trader/syncloop package: shared stoppable sync loop with exponential failure backoff (30s base, 5min cap) - all 9 exchanges' StartOrderSync now take the trader's stop channel and stop when the trader stops (close broadcast from AutoTrader.Stop) - provider/hyperliquid: GetPerpDexCoins now serves a 5min TTL cache and falls back to the stale board when the upstream returns 429, so the symbol panel keeps working through rate limiting
This commit is contained in:
48
trader/syncloop/syncloop.go
Normal file
48
trader/syncloop/syncloop.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Package syncloop runs background exchange order-sync loops with a shared
|
||||
// lifecycle: loops stop when the owning trader stops, and consecutive
|
||||
// failures back off exponentially so a rate-limiting exchange (HTTP 429)
|
||||
// is not hammered at full frequency.
|
||||
package syncloop
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"nofx/logger"
|
||||
)
|
||||
|
||||
// maxBackoff caps the failure backoff so a recovered exchange is picked up
|
||||
// within a few minutes at worst.
|
||||
const maxBackoff = 5 * time.Minute
|
||||
|
||||
// Run starts a background loop calling syncFn at the given interval until
|
||||
// stop is closed. After each consecutive failure the wait doubles (capped at
|
||||
// maxBackoff); the first success resets it to the base interval.
|
||||
func Run(stop <-chan struct{}, interval time.Duration, name string, syncFn func() error) {
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
go func() {
|
||||
wait := interval
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
logger.Infof("⏹ %s order sync stopped", name)
|
||||
return
|
||||
case <-timer.C:
|
||||
if err := syncFn(); err != nil {
|
||||
wait *= 2
|
||||
if wait > maxBackoff {
|
||||
wait = maxBackoff
|
||||
}
|
||||
logger.Infof("⚠️ %s order sync failed: %v (backing off, next attempt in %v)", name, err, wait)
|
||||
} else {
|
||||
wait = interval
|
||||
}
|
||||
timer.Reset(wait)
|
||||
}
|
||||
}
|
||||
}()
|
||||
logger.Infof("🔄 %s order sync started (interval: %v)", name, interval)
|
||||
}
|
||||
78
trader/syncloop/syncloop_test.go
Normal file
78
trader/syncloop/syncloop_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package syncloop
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunStopsWhenStopChannelCloses(t *testing.T) {
|
||||
stop := make(chan struct{})
|
||||
var calls atomic.Int64
|
||||
|
||||
Run(stop, 5*time.Millisecond, "test", func() error {
|
||||
calls.Add(1)
|
||||
return nil
|
||||
})
|
||||
|
||||
// Let a few ticks happen, then stop.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
close(stop)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
after := calls.Load()
|
||||
if after == 0 {
|
||||
t.Fatal("sync function never ran")
|
||||
}
|
||||
|
||||
// No further calls after stop.
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
if calls.Load() != after {
|
||||
t.Fatalf("sync kept running after stop: %d -> %d", after, calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBacksOffOnConsecutiveFailures(t *testing.T) {
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
var calls atomic.Int64
|
||||
|
||||
Run(stop, 10*time.Millisecond, "test", func() error {
|
||||
calls.Add(1)
|
||||
return errors.New("API returned status 429")
|
||||
})
|
||||
|
||||
// With a 10ms base interval and exponential backoff (10, 20, 40, 80...),
|
||||
// 100ms allows at most ~4 failing attempts. Without backoff there would
|
||||
// be ~10.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
got := calls.Load()
|
||||
if got == 0 {
|
||||
t.Fatal("sync function never ran")
|
||||
}
|
||||
if got > 5 {
|
||||
t.Fatalf("expected backoff to throttle failing sync, got %d calls in 100ms", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecoversIntervalAfterSuccess(t *testing.T) {
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
var calls atomic.Int64
|
||||
|
||||
// Fail twice, then succeed forever.
|
||||
Run(stop, 5*time.Millisecond, "test", func() error {
|
||||
n := calls.Add(1)
|
||||
if n <= 2 {
|
||||
return errors.New("transient")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Failures at 5ms and 15ms (backoff 10ms), success at ~35ms (backoff 20ms),
|
||||
// then the interval resets to 5ms — plenty of successful runs by 150ms.
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if got := calls.Load(); got < 8 {
|
||||
t.Fatalf("expected interval to reset after success, got only %d calls", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user