feat: server-side launch preflight with structured checks and runtime AI wallet health

- POST /api/launch/preflight and GET /api/traders/:id/preflight run every
  launch prerequisite (AI model credential, claw402 wallet key + Base USDC
  balance, strategy, exchange config, live account state, funding minimums)
  and return a structured checklist with stable codes; minimums (1 USDC AI
  fee / 12 USDC trading) now live server-side and are exposed in the response.
- POST /traders/:id/start enforces the preflight (400 with error_key
  trader.start.preflight_failed + full checklist; ?force=true to override).
  Funding gates use max(available, equity) so restarting a bot with deployed
  margin is not blocked; indeterminate probes (RPC outage) degrade to
  warnings instead of blocking.
- Run loop now surfaces runtime health via GetStatus: safe_mode(+reason) and
  ai_wallet_status/balance (ok|low|empty|unknown), including typed detection
  of claw402 ErrInsufficientFunds instead of burying it in logs; safe-mode
  state is mutex-guarded for API readers.
- Onboarding wallet endpoints switch to the cached, error-aware balance query
  and report balance_status so the UI can tell an RPC outage from an empty
  wallet.
This commit is contained in:
tinkle-community
2026-07-05 20:38:38 +09:00
parent e61f12d307
commit b17d97effc
10 changed files with 843 additions and 14 deletions

View File

@@ -177,8 +177,12 @@ type AutoTrader struct {
gridState *GridState // Grid trading state (only used when StrategyType == "grid_trading")
claw402WalletAddr string // Claw402 wallet address (derived from private key at start)
consecutiveAIFailures int // Consecutive AI call failures
runtimeHealthMu sync.RWMutex // Guards safe mode + AI wallet health (loop writes, API reads)
safeMode bool // Safe mode: no new positions, protect existing ones
safeModeReason string // Why safe mode was activated
aiWalletStatus string // "ok"|"low"|"empty"|"unknown" — see runtime_health.go
aiWalletBalanceUSDC float64 // Last observed Base USDC balance of the claw402 wallet
aiWalletCheckedAt time.Time // When the balance was last observed
}
// NewAutoTrader creates an automatic trader
@@ -710,7 +714,9 @@ func (at *AutoTrader) runPreLaunchChecks() {
balance, err := wallet.QueryUSDCBalance(addr)
if err != nil {
logger.Warnf("⚠️ [%s] Could not query USDC balance: %v", at.name, err)
at.markAIWalletHealthUnknown()
} else {
at.setAIWalletHealth(balance)
// Estimate runway
scanMinutes := int(at.config.ScanInterval.Minutes())
modelName := at.config.CustomModelName

View File

@@ -91,6 +91,19 @@ func (at *AutoTrader) GetStatus() map[string]interface{} {
}
}
// Runtime health: safe mode + AI fee wallet, so the dashboard can show a
// persistent banner instead of the user digging through logs.
safeMode, safeModeReason := at.safeModeState()
result["safe_mode"] = safeMode
if safeModeReason != "" {
result["safe_mode_reason"] = safeModeReason
}
if status, balance, checkedAt := at.aiWalletHealth(); status != "" {
result["ai_wallet_status"] = status
result["ai_wallet_balance_usdc"] = balance
result["ai_wallet_checked_at"] = checkedAt.Format(time.RFC3339)
}
return result
}

View File

@@ -21,7 +21,7 @@ import (
// - never exceeds MaxPositions,
// - never doubles a base symbol already held or already in the decision set.
func (at *AutoTrader) ensureLongShortCoverage(decisions []kernel.Decision, ctx *kernel.Context, equity float64) []kernel.Decision {
if at == nil || ctx == nil || at.safeMode {
if at == nil || ctx == nil || at.isSafeMode() {
return decisions
}
if at.config.StrategyConfig == nil || at.config.StrategyConfig.CoinSource.SourceType != "vergex_signal" {

View File

@@ -2,10 +2,12 @@ package trader
import (
"encoding/json"
"errors"
"fmt"
"nofx/kernel"
"nofx/logger"
"nofx/market"
"nofx/mcp/payment"
"nofx/provider/hyperliquid"
"nofx/store"
"nofx/wallet"
@@ -141,10 +143,21 @@ func (at *AutoTrader) runCycle() error {
record.Success = false
record.ErrorMessage = fmt.Sprintf("Failed to get AI decision: %v", err)
// Payment-layer rejection means the AI fee wallet is definitively out
// of funds — surface it as structured health state (GetStatus), not
// just a log line.
var insufficientFunds *payment.ErrInsufficientFunds
if errors.As(err, &insufficientFunds) {
at.markAIWalletEmptyFromPayment(insufficientFunds.Balance)
record.ErrorMessage = fmt.Sprintf(
"AI fee wallet out of funds: balance $%.2f USDC, next call needs ~$%.2f. Top up the Base USDC wallet.",
insufficientFunds.Balance, insufficientFunds.Needed,
)
}
// Activate safe mode after 3 consecutive failures
if at.consecutiveAIFailures >= 3 && !at.safeMode {
at.safeMode = true
at.safeModeReason = fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err)
if at.consecutiveAIFailures >= 3 && !at.isSafeMode() {
at.setSafeMode(true, fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err))
at.logErrorf("🛡️ SAFE MODE ACTIVATED — AI failed %d times in a row. No new positions will be opened. Existing positions are protected with current stop-loss settings.", at.consecutiveAIFailures)
at.logErrorf("🛡️ Reason: %v", err)
at.logErrorf("🛡️ Action: Will keep trying AI each cycle. Safe mode auto-deactivates when AI recovers.")
@@ -172,7 +185,7 @@ func (at *AutoTrader) runCycle() error {
}
// In safe mode, don't return error — keep the loop running to retry next cycle
if at.safeMode {
if at.isSafeMode() {
at.logWarnf("🛡️ Safe mode: skipping this cycle, will retry in %v", at.config.ScanInterval)
return nil
}
@@ -185,10 +198,9 @@ func (at *AutoTrader) runCycle() error {
at.logInfof("✅ AI recovered after %d consecutive failures", at.consecutiveAIFailures)
}
at.consecutiveAIFailures = 0
if at.safeMode {
if at.isSafeMode() {
at.logInfof("🛡️ SAFE MODE DEACTIVATED — AI is working again. Resuming normal trading.")
at.safeMode = false
at.safeModeReason = ""
at.setSafeMode(false, "")
}
// // 5. Print system prompt
@@ -242,7 +254,7 @@ func (at *AutoTrader) runCycle() error {
}
// Safe mode: filter out open positions, only allow close/hold
if at.safeMode {
if at.isSafeMode() {
filtered := make([]kernel.Decision, 0)
for _, d := range sortedDecisions {
if d.Action == "open_long" || d.Action == "open_short" {
@@ -770,10 +782,12 @@ func (at *AutoTrader) checkClaw402Balance() {
balance, err := wallet.QueryUSDCBalance(at.claw402WalletAddr)
if err != nil {
at.logWarnf("⚠️ Failed to query USDC balance: %v", err)
at.markAIWalletHealthUnknown()
return
}
if balance < 1.0 {
at.setAIWalletHealth(balance)
if balance < aiWalletLowThresholdUSDC {
at.logWarnf("⚠️ Low USDC balance: $%.2f — AI may stop soon!", balance)
}
if balance <= 0 {

82
trader/runtime_health.go Normal file
View File

@@ -0,0 +1,82 @@
package trader
import (
"time"
)
// Runtime health state written by the run-loop goroutine and read by the API
// layer (GetStatus). Everything here goes through runtimeHealthMu so the
// dashboard can poll without racing the trading loop.
// AI fee wallet health statuses exposed via GetStatus as "ai_wallet_status".
const (
AIWalletStatusOK = "ok"
AIWalletStatusLow = "low"
AIWalletStatusEmpty = "empty"
AIWalletStatusUnknown = "unknown"
)
// aiWalletLowThresholdUSDC mirrors api.MinAIFeeUSDC: below this the wallet
// cannot reliably pay for the next AI/data calls.
const aiWalletLowThresholdUSDC = 1.0
func (at *AutoTrader) setSafeMode(active bool, reason string) {
at.runtimeHealthMu.Lock()
at.safeMode = active
at.safeModeReason = reason
at.runtimeHealthMu.Unlock()
}
func (at *AutoTrader) isSafeMode() bool {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.safeMode
}
func (at *AutoTrader) safeModeState() (bool, string) {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.safeMode, at.safeModeReason
}
// setAIWalletHealth records a fresh balance observation for the claw402 wallet.
func (at *AutoTrader) setAIWalletHealth(balance float64) {
status := AIWalletStatusOK
switch {
case balance <= 0:
status = AIWalletStatusEmpty
case balance < aiWalletLowThresholdUSDC:
status = AIWalletStatusLow
}
at.runtimeHealthMu.Lock()
at.aiWalletStatus = status
at.aiWalletBalanceUSDC = balance
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
// markAIWalletHealthUnknown keeps the last observed balance but flags that the
// current reading could not be refreshed (e.g. Base RPC unreachable).
func (at *AutoTrader) markAIWalletHealthUnknown() {
at.runtimeHealthMu.Lock()
at.aiWalletStatus = AIWalletStatusUnknown
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
// markAIWalletEmptyFromPayment records a payment-layer rejection: the wallet
// definitively could not cover an AI call.
func (at *AutoTrader) markAIWalletEmptyFromPayment(balance float64) {
at.runtimeHealthMu.Lock()
at.aiWalletStatus = AIWalletStatusEmpty
at.aiWalletBalanceUSDC = balance
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
func (at *AutoTrader) aiWalletHealth() (status string, balance float64, checkedAt time.Time) {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.aiWalletStatus, at.aiWalletBalanceUSDC, at.aiWalletCheckedAt
}