mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-11 14:56:57 +08:00
feat: server-side launch preflight with structured checks and runtime AI wallet health
- POST /api/launch/preflight and GET /api/traders/:id/preflight run every launch prerequisite (AI model credential, claw402 wallet key + Base USDC balance, strategy, exchange config, live account state, funding minimums) and return a structured checklist with stable codes; minimums (1 USDC AI fee / 12 USDC trading) now live server-side and are exposed in the response. - POST /traders/:id/start enforces the preflight (400 with error_key trader.start.preflight_failed + full checklist; ?force=true to override). Funding gates use max(available, equity) so restarting a bot with deployed margin is not blocked; indeterminate probes (RPC outage) degrade to warnings instead of blocking. - Run loop now surfaces runtime health via GetStatus: safe_mode(+reason) and ai_wallet_status/balance (ok|low|empty|unknown), including typed detection of claw402 ErrInsufficientFunds instead of burying it in logs; safe-mode state is mutex-guarded for API readers. - Onboarding wallet endpoints switch to the cached, error-aware balance query and report balance_status so the UI can tell an RPC outage from an empty wallet.
This commit is contained in:
@@ -177,8 +177,12 @@ type AutoTrader struct {
|
||||
gridState *GridState // Grid trading state (only used when StrategyType == "grid_trading")
|
||||
claw402WalletAddr string // Claw402 wallet address (derived from private key at start)
|
||||
consecutiveAIFailures int // Consecutive AI call failures
|
||||
runtimeHealthMu sync.RWMutex // Guards safe mode + AI wallet health (loop writes, API reads)
|
||||
safeMode bool // Safe mode: no new positions, protect existing ones
|
||||
safeModeReason string // Why safe mode was activated
|
||||
aiWalletStatus string // "ok"|"low"|"empty"|"unknown" — see runtime_health.go
|
||||
aiWalletBalanceUSDC float64 // Last observed Base USDC balance of the claw402 wallet
|
||||
aiWalletCheckedAt time.Time // When the balance was last observed
|
||||
}
|
||||
|
||||
// NewAutoTrader creates an automatic trader
|
||||
@@ -710,7 +714,9 @@ func (at *AutoTrader) runPreLaunchChecks() {
|
||||
balance, err := wallet.QueryUSDCBalance(addr)
|
||||
if err != nil {
|
||||
logger.Warnf("⚠️ [%s] Could not query USDC balance: %v", at.name, err)
|
||||
at.markAIWalletHealthUnknown()
|
||||
} else {
|
||||
at.setAIWalletHealth(balance)
|
||||
// Estimate runway
|
||||
scanMinutes := int(at.config.ScanInterval.Minutes())
|
||||
modelName := at.config.CustomModelName
|
||||
|
||||
@@ -91,6 +91,19 @@ func (at *AutoTrader) GetStatus() map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime health: safe mode + AI fee wallet, so the dashboard can show a
|
||||
// persistent banner instead of the user digging through logs.
|
||||
safeMode, safeModeReason := at.safeModeState()
|
||||
result["safe_mode"] = safeMode
|
||||
if safeModeReason != "" {
|
||||
result["safe_mode_reason"] = safeModeReason
|
||||
}
|
||||
if status, balance, checkedAt := at.aiWalletHealth(); status != "" {
|
||||
result["ai_wallet_status"] = status
|
||||
result["ai_wallet_balance_usdc"] = balance
|
||||
result["ai_wallet_checked_at"] = checkedAt.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// - never exceeds MaxPositions,
|
||||
// - never doubles a base symbol already held or already in the decision set.
|
||||
func (at *AutoTrader) ensureLongShortCoverage(decisions []kernel.Decision, ctx *kernel.Context, equity float64) []kernel.Decision {
|
||||
if at == nil || ctx == nil || at.safeMode {
|
||||
if at == nil || ctx == nil || at.isSafeMode() {
|
||||
return decisions
|
||||
}
|
||||
if at.config.StrategyConfig == nil || at.config.StrategyConfig.CoinSource.SourceType != "vergex_signal" {
|
||||
|
||||
@@ -2,10 +2,12 @@ package trader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"nofx/kernel"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/mcp/payment"
|
||||
"nofx/provider/hyperliquid"
|
||||
"nofx/store"
|
||||
"nofx/wallet"
|
||||
@@ -141,10 +143,21 @@ func (at *AutoTrader) runCycle() error {
|
||||
record.Success = false
|
||||
record.ErrorMessage = fmt.Sprintf("Failed to get AI decision: %v", err)
|
||||
|
||||
// Payment-layer rejection means the AI fee wallet is definitively out
|
||||
// of funds — surface it as structured health state (GetStatus), not
|
||||
// just a log line.
|
||||
var insufficientFunds *payment.ErrInsufficientFunds
|
||||
if errors.As(err, &insufficientFunds) {
|
||||
at.markAIWalletEmptyFromPayment(insufficientFunds.Balance)
|
||||
record.ErrorMessage = fmt.Sprintf(
|
||||
"AI fee wallet out of funds: balance $%.2f USDC, next call needs ~$%.2f. Top up the Base USDC wallet.",
|
||||
insufficientFunds.Balance, insufficientFunds.Needed,
|
||||
)
|
||||
}
|
||||
|
||||
// Activate safe mode after 3 consecutive failures
|
||||
if at.consecutiveAIFailures >= 3 && !at.safeMode {
|
||||
at.safeMode = true
|
||||
at.safeModeReason = fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err)
|
||||
if at.consecutiveAIFailures >= 3 && !at.isSafeMode() {
|
||||
at.setSafeMode(true, fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err))
|
||||
at.logErrorf("🛡️ SAFE MODE ACTIVATED — AI failed %d times in a row. No new positions will be opened. Existing positions are protected with current stop-loss settings.", at.consecutiveAIFailures)
|
||||
at.logErrorf("🛡️ Reason: %v", err)
|
||||
at.logErrorf("🛡️ Action: Will keep trying AI each cycle. Safe mode auto-deactivates when AI recovers.")
|
||||
@@ -172,7 +185,7 @@ func (at *AutoTrader) runCycle() error {
|
||||
}
|
||||
|
||||
// In safe mode, don't return error — keep the loop running to retry next cycle
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
at.logWarnf("🛡️ Safe mode: skipping this cycle, will retry in %v", at.config.ScanInterval)
|
||||
return nil
|
||||
}
|
||||
@@ -185,10 +198,9 @@ func (at *AutoTrader) runCycle() error {
|
||||
at.logInfof("✅ AI recovered after %d consecutive failures", at.consecutiveAIFailures)
|
||||
}
|
||||
at.consecutiveAIFailures = 0
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
at.logInfof("🛡️ SAFE MODE DEACTIVATED — AI is working again. Resuming normal trading.")
|
||||
at.safeMode = false
|
||||
at.safeModeReason = ""
|
||||
at.setSafeMode(false, "")
|
||||
}
|
||||
|
||||
// // 5. Print system prompt
|
||||
@@ -242,7 +254,7 @@ func (at *AutoTrader) runCycle() error {
|
||||
}
|
||||
|
||||
// Safe mode: filter out open positions, only allow close/hold
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
filtered := make([]kernel.Decision, 0)
|
||||
for _, d := range sortedDecisions {
|
||||
if d.Action == "open_long" || d.Action == "open_short" {
|
||||
@@ -770,10 +782,12 @@ func (at *AutoTrader) checkClaw402Balance() {
|
||||
balance, err := wallet.QueryUSDCBalance(at.claw402WalletAddr)
|
||||
if err != nil {
|
||||
at.logWarnf("⚠️ Failed to query USDC balance: %v", err)
|
||||
at.markAIWalletHealthUnknown()
|
||||
return
|
||||
}
|
||||
|
||||
if balance < 1.0 {
|
||||
at.setAIWalletHealth(balance)
|
||||
if balance < aiWalletLowThresholdUSDC {
|
||||
at.logWarnf("⚠️ Low USDC balance: $%.2f — AI may stop soon!", balance)
|
||||
}
|
||||
if balance <= 0 {
|
||||
|
||||
82
trader/runtime_health.go
Normal file
82
trader/runtime_health.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package trader
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Runtime health state written by the run-loop goroutine and read by the API
|
||||
// layer (GetStatus). Everything here goes through runtimeHealthMu so the
|
||||
// dashboard can poll without racing the trading loop.
|
||||
|
||||
// AI fee wallet health statuses exposed via GetStatus as "ai_wallet_status".
|
||||
const (
|
||||
AIWalletStatusOK = "ok"
|
||||
AIWalletStatusLow = "low"
|
||||
AIWalletStatusEmpty = "empty"
|
||||
AIWalletStatusUnknown = "unknown"
|
||||
)
|
||||
|
||||
// aiWalletLowThresholdUSDC mirrors api.MinAIFeeUSDC: below this the wallet
|
||||
// cannot reliably pay for the next AI/data calls.
|
||||
const aiWalletLowThresholdUSDC = 1.0
|
||||
|
||||
func (at *AutoTrader) setSafeMode(active bool, reason string) {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.safeMode = active
|
||||
at.safeModeReason = reason
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
func (at *AutoTrader) isSafeMode() bool {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.safeMode
|
||||
}
|
||||
|
||||
func (at *AutoTrader) safeModeState() (bool, string) {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.safeMode, at.safeModeReason
|
||||
}
|
||||
|
||||
// setAIWalletHealth records a fresh balance observation for the claw402 wallet.
|
||||
func (at *AutoTrader) setAIWalletHealth(balance float64) {
|
||||
status := AIWalletStatusOK
|
||||
switch {
|
||||
case balance <= 0:
|
||||
status = AIWalletStatusEmpty
|
||||
case balance < aiWalletLowThresholdUSDC:
|
||||
status = AIWalletStatusLow
|
||||
}
|
||||
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = status
|
||||
at.aiWalletBalanceUSDC = balance
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
// markAIWalletHealthUnknown keeps the last observed balance but flags that the
|
||||
// current reading could not be refreshed (e.g. Base RPC unreachable).
|
||||
func (at *AutoTrader) markAIWalletHealthUnknown() {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = AIWalletStatusUnknown
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
// markAIWalletEmptyFromPayment records a payment-layer rejection: the wallet
|
||||
// definitively could not cover an AI call.
|
||||
func (at *AutoTrader) markAIWalletEmptyFromPayment(balance float64) {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = AIWalletStatusEmpty
|
||||
at.aiWalletBalanceUSDC = balance
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
func (at *AutoTrader) aiWalletHealth() (status string, balance float64, checkedAt time.Time) {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.aiWalletStatus, at.aiWalletBalanceUSDC, at.aiWalletCheckedAt
|
||||
}
|
||||
Reference in New Issue
Block a user