feat: server-side launch preflight with structured checks and runtime AI wallet health

- POST /api/launch/preflight and GET /api/traders/:id/preflight run every
  launch prerequisite (AI model credential, claw402 wallet key + Base USDC
  balance, strategy, exchange config, live account state, funding minimums)
  and return a structured checklist with stable codes; minimums (1 USDC AI
  fee / 12 USDC trading) now live server-side and are exposed in the response.
- POST /traders/:id/start enforces the preflight (400 with error_key
  trader.start.preflight_failed + full checklist; ?force=true to override).
  Funding gates use max(available, equity) so restarting a bot with deployed
  margin is not blocked; indeterminate probes (RPC outage) degrade to
  warnings instead of blocking.
- Run loop now surfaces runtime health via GetStatus: safe_mode(+reason) and
  ai_wallet_status/balance (ok|low|empty|unknown), including typed detection
  of claw402 ErrInsufficientFunds instead of burying it in logs; safe-mode
  state is mutex-guarded for API readers.
- Onboarding wallet endpoints switch to the cached, error-aware balance query
  and report balance_status so the UI can tell an RPC outage from an empty
  wallet.
This commit is contained in:
tinkle-community
2026-07-05 20:38:38 +09:00
parent e61f12d307
commit b17d97effc
10 changed files with 843 additions and 14 deletions

View File

@@ -26,6 +26,7 @@ type beginnerOnboardingResponse struct {
DefaultModel string `json:"default_model"`
ConfiguredModelID string `json:"configured_model_id"`
BalanceUSDC string `json:"balance_usdc"`
BalanceStatus string `json:"balance_status,omitempty"`
EnvSaved bool `json:"env_saved"`
EnvPath string `json:"env_path,omitempty"`
ReusedExisting bool `json:"reused_existing"`
@@ -36,10 +37,22 @@ type currentBeginnerWalletResponse struct {
Found bool `json:"found"`
Address string `json:"address,omitempty"`
BalanceUSDC string `json:"balance_usdc,omitempty"`
BalanceStatus string `json:"balance_status,omitempty"`
Source string `json:"source,omitempty"`
Claw402Status string `json:"claw402_status"`
}
// queryBeginnerWalletBalance returns the wallet balance plus a status flag so
// the UI can distinguish "RPC unreachable" from a genuinely empty wallet.
// Uses the 30s cache — safe for UI polling.
func queryBeginnerWalletBalance(address string) (balanceUSDC string, balanceStatus string) {
balance, err := wallet.QueryUSDCBalanceCached(address)
if err != nil {
return "", "unknown"
}
return fmt.Sprintf("%.2f", balance), "ok"
}
func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
userID := c.GetString("user_id")
if userID == "" {
@@ -72,6 +85,7 @@ func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
os.Setenv("CLAW402_DEFAULT_MODEL", payment.DefaultClaw402Model)
envSaved, envPath, envErr := persistBeginnerWalletEnv(privateKey, address)
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
resp := beginnerOnboardingResponse{
Address: address,
PrivateKey: privateKey,
@@ -80,7 +94,8 @@ func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
Provider: "claw402",
DefaultModel: payment.DefaultClaw402Model,
ConfiguredModelID: configuredModelID,
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
BalanceUSDC: balanceUSDC,
BalanceStatus: balanceStatus,
EnvSaved: envSaved,
EnvPath: envPath,
ReusedExisting: reusedExisting,
@@ -124,10 +139,12 @@ func (s *Server) handleCurrentBeginnerWallet(c *gin.Context) {
continue
}
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
c.JSON(http.StatusOK, currentBeginnerWalletResponse{
Found: true,
Address: address,
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
BalanceUSDC: balanceUSDC,
BalanceStatus: balanceStatus,
Source: "model",
Claw402Status: claw402Status,
})
@@ -136,10 +153,12 @@ func (s *Server) handleCurrentBeginnerWallet(c *gin.Context) {
address := strings.TrimSpace(os.Getenv("CLAW402_WALLET_ADDRESS"))
if address != "" {
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
c.JSON(http.StatusOK, currentBeginnerWalletResponse{
Found: true,
Address: address,
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
BalanceUSDC: balanceUSDC,
BalanceStatus: balanceStatus,
Source: "env",
Claw402Status: claw402Status,
})

View File

@@ -850,6 +850,24 @@ func (s *Server) handleStartTrader(c *gin.Context) {
return
}
// Server-side launch gate: the trader cannot function without a funded AI
// wallet and a ready exchange account, so verify both before the run loop
// starts. `?force=true` skips the gate for deliberate manual overrides.
if c.Query("force") != "true" {
// strategyRequired=false: a trader that loaded into memory necessarily
// has a valid strategy (the manager refuses to load without one), so the
// preflight strategy check would be redundant here.
preflight := s.runLaunchPreflight(userID, fullCfg.AIModel, fullCfg.Exchange, fullCfg.Strategy, false)
if !preflight.Ready {
c.JSON(http.StatusBadRequest, gin.H{
"error": formatTraderStartError(preflight.Summary(), "Complete the failing checks, then start the bot again"),
"error_key": "trader.start.preflight_failed",
"preflight": preflight,
})
return
}
}
// Start trader
go func() {
logger.Infof("▶️ Starting trader %s (%s)", traderID, trader.GetName())

375
api/launch_preflight.go Normal file
View File

@@ -0,0 +1,375 @@
package api
import (
"fmt"
"net/http"
"strings"
"time"
"nofx/store"
"nofx/wallet"
"github.com/gin-gonic/gin"
)
// Launch readiness minimums. These are the single source of truth — the
// frontend reads them from the preflight response instead of hardcoding.
const (
// MinAIFeeUSDC is the minimum Base USDC a claw402 fee wallet needs so the
// trader can pay for its first AI/data calls.
MinAIFeeUSDC = 1.0
// MinTradingUSDC is the minimum available balance the exchange account
// needs before the trader can place its first order.
MinTradingUSDC = 12.0
)
const (
launchCheckStatusOK = "ok"
launchCheckStatusFailed = "failed"
launchCheckStatusWarning = "warning"
launchCheckStatusSkipped = "skipped"
)
// Check IDs — stable identifiers the frontend maps to guided-setup steps.
const (
launchCheckAIModel = "ai_model"
launchCheckAIWallet = "ai_wallet"
launchCheckAIWalletFunds = "ai_wallet_funds"
launchCheckStrategy = "strategy"
launchCheckExchangeConfig = "exchange_config"
launchCheckExchangeAccount = "exchange_account"
launchCheckExchangeFunds = "exchange_funds"
)
// queryAIWalletBalance is swappable in tests to avoid live Base RPC calls.
var queryAIWalletBalance = wallet.QueryUSDCBalanceCached
type LaunchCheck struct {
ID string `json:"id"`
Status string `json:"status"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
// Numeric context so the UI can render progress like "8.40 / 12 USDC".
Required float64 `json:"required,omitempty"`
Actual *float64 `json:"actual,omitempty"`
Asset string `json:"asset,omitempty"`
// Address is the funding address for balance checks, so the UI can offer
// a deposit shortcut next to the failing item.
Address string `json:"address,omitempty"`
}
type LaunchPreflightResult struct {
Ready bool `json:"ready"`
Checks []LaunchCheck `json:"checks"`
MinAIFeeUSDC float64 `json:"min_ai_fee_usdc"`
MinTradingUSDC float64 `json:"min_trading_usdc"`
CheckedAt time.Time `json:"checked_at"`
}
func (r LaunchPreflightResult) failedChecks() []LaunchCheck {
var failed []LaunchCheck
for _, check := range r.Checks {
if check.Status == launchCheckStatusFailed {
failed = append(failed, check)
}
}
return failed
}
// Summary joins the failing messages into one human-readable sentence.
func (r LaunchPreflightResult) Summary() string {
failed := r.failedChecks()
if len(failed) == 0 {
return ""
}
messages := make([]string, 0, len(failed))
for _, check := range failed {
if check.Message != "" {
messages = append(messages, check.Message)
}
}
return strings.Join(messages, " ")
}
type launchPreflightRequest struct {
AIModelID string `json:"ai_model_id"`
ExchangeID string `json:"exchange_id"`
StrategyID string `json:"strategy_id"`
}
// handleLaunchPreflight runs launch readiness checks for a model/exchange
// pair before any trader is created or mutated. POST /api/launch/preflight
func (s *Server) handleLaunchPreflight(c *gin.Context) {
userID := c.GetString("user_id")
var req launchPreflightRequest
if err := c.ShouldBindJSON(&req); err != nil {
SafeBadRequest(c, "Invalid preflight request")
return
}
if strings.TrimSpace(req.AIModelID) == "" || strings.TrimSpace(req.ExchangeID) == "" {
SafeBadRequest(c, "ai_model_id and exchange_id are required")
return
}
model, err := s.store.AIModel().Get(userID, req.AIModelID)
if err != nil {
model = nil
}
exchange, err := s.store.Exchange().GetByID(userID, req.ExchangeID)
if err != nil {
exchange = nil
}
var strategy *store.Strategy
strategyRequired := strings.TrimSpace(req.StrategyID) != ""
if strategyRequired {
strategy, err = s.store.Strategy().Get(userID, req.StrategyID)
if err != nil {
strategy = nil
}
}
result := s.runLaunchPreflight(userID, model, exchange, strategy, strategyRequired)
c.JSON(http.StatusOK, result)
}
// handleTraderPreflight runs the same checks against an existing trader's
// full configuration. GET /api/traders/:id/preflight
func (s *Server) handleTraderPreflight(c *gin.Context) {
userID := c.GetString("user_id")
traderID := c.Param("id")
fullCfg, err := s.store.Trader().GetFullConfig(userID, traderID)
if err != nil || fullCfg == nil || fullCfg.Trader == nil {
SafeNotFound(c, "Trader")
return
}
result := s.runLaunchPreflight(userID, fullCfg.AIModel, fullCfg.Exchange, fullCfg.Strategy, true)
c.JSON(http.StatusOK, result)
}
// runLaunchPreflight composes every launch readiness check. A check that
// cannot be evaluated (RPC outage, probe timeout) reports "warning" instead
// of "failed" so an infrastructure hiccup never hard-blocks a launch.
func (s *Server) runLaunchPreflight(
userID string,
model *store.AIModel,
exchange *store.Exchange,
strategy *store.Strategy,
strategyRequired bool,
) LaunchPreflightResult {
checks := []LaunchCheck{checkLaunchAIModel(model)}
checks = append(checks, checkLaunchAIWallet(model)...)
checks = append(checks, checkLaunchStrategy(strategy, strategyRequired))
checks = append(checks, s.checkLaunchExchange(userID, exchange)...)
ready := true
for _, check := range checks {
if check.Status == launchCheckStatusFailed {
ready = false
break
}
}
return LaunchPreflightResult{
Ready: ready,
Checks: checks,
MinAIFeeUSDC: MinAIFeeUSDC,
MinTradingUSDC: MinTradingUSDC,
CheckedAt: time.Now().UTC(),
}
}
func checkLaunchAIModel(model *store.AIModel) LaunchCheck {
check := LaunchCheck{ID: launchCheckAIModel}
switch {
case model == nil:
check.Status = launchCheckStatusFailed
check.Code = "MODEL_NOT_FOUND"
check.Message = "The selected AI model was not found. Configure an AI model first."
case !model.Enabled:
check.Status = launchCheckStatusFailed
check.Code = "MODEL_DISABLED"
check.Message = fmt.Sprintf("AI model \"%s\" is disabled. Enable it first.", model.Name)
case strings.TrimSpace(model.APIKey.String()) == "":
check.Status = launchCheckStatusFailed
check.Code = "MODEL_MISSING_CREDENTIALS"
check.Message = fmt.Sprintf("AI model \"%s\" has no credential saved. Add the API key or wallet key first.", model.Name)
default:
check.Status = launchCheckStatusOK
check.Message = model.Name
}
return check
}
// checkLaunchAIWallet validates the claw402 fee wallet (address + Base USDC
// balance). Non-claw402 providers pay per API key, so both checks are skipped.
func checkLaunchAIWallet(model *store.AIModel) []LaunchCheck {
walletCheck := LaunchCheck{ID: launchCheckAIWallet}
fundsCheck := LaunchCheck{ID: launchCheckAIWalletFunds, Asset: "USDC", Required: MinAIFeeUSDC}
if model == nil || model.Provider != "claw402" || strings.TrimSpace(model.APIKey.String()) == "" {
walletCheck.Status = launchCheckStatusSkipped
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{walletCheck, fundsCheck}
}
address, err := walletAddressFromPrivateKey(model.APIKey.String())
if err != nil {
walletCheck.Status = launchCheckStatusFailed
walletCheck.Code = "AI_WALLET_INVALID_KEY"
walletCheck.Message = "The Claw402 wallet key is invalid. Recreate the Base USDC payment wallet."
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{walletCheck, fundsCheck}
}
walletCheck.Status = launchCheckStatusOK
walletCheck.Address = address
fundsCheck.Address = address
balance, err := queryAIWalletBalance(address)
if err != nil {
fundsCheck.Status = launchCheckStatusWarning
fundsCheck.Code = "AI_WALLET_BALANCE_UNKNOWN"
fundsCheck.Message = "Could not verify the Base USDC balance right now. The trader will start, but AI calls fail if the wallet is empty."
return []LaunchCheck{walletCheck, fundsCheck}
}
fundsCheck.Actual = &balance
if balance < MinAIFeeUSDC {
fundsCheck.Status = launchCheckStatusFailed
fundsCheck.Code = "AI_WALLET_INSUFFICIENT_FUNDS"
fundsCheck.Message = fmt.Sprintf(
"The Claw402 wallet holds %.2f USDC but needs at least %.0f USDC on Base to pay for AI and data calls.",
balance, MinAIFeeUSDC,
)
} else {
fundsCheck.Status = launchCheckStatusOK
}
return []LaunchCheck{walletCheck, fundsCheck}
}
func checkLaunchStrategy(strategy *store.Strategy, required bool) LaunchCheck {
check := LaunchCheck{ID: launchCheckStrategy}
switch {
case strategy != nil:
check.Status = launchCheckStatusOK
check.Message = strategy.Name
case required:
check.Status = launchCheckStatusFailed
check.Code = "STRATEGY_NOT_FOUND"
check.Message = "The selected strategy was not found. Pick or create a strategy first."
default:
check.Status = launchCheckStatusSkipped
}
return check
}
// checkLaunchExchange validates exchange configuration completeness and then
// probes the live account (30s server cache) for status and balance.
func (s *Server) checkLaunchExchange(userID string, exchange *store.Exchange) []LaunchCheck {
configCheck := LaunchCheck{ID: launchCheckExchangeConfig}
accountCheck := LaunchCheck{ID: launchCheckExchangeAccount}
fundsCheck := LaunchCheck{ID: launchCheckExchangeFunds, Required: MinTradingUSDC}
if msg, code := describeExchangeConfigIssue(exchange); code != "" {
configCheck.Status = launchCheckStatusFailed
configCheck.Code = code
configCheck.Message = msg
accountCheck.Status = launchCheckStatusSkipped
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
}
configCheck.Status = launchCheckStatusOK
configCheck.Message = exchangeDisplayName(exchange)
fundsCheck.Asset = accountAssetForExchange(exchange.ExchangeType)
states, err := s.getExchangeAccountStates(userID)
if err != nil {
accountCheck.Status = launchCheckStatusWarning
accountCheck.Code = "EXCHANGE_STATE_UNKNOWN"
accountCheck.Message = "Could not verify the exchange account right now."
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
}
state, ok := states[exchange.ID]
if !ok {
accountCheck.Status = launchCheckStatusWarning
accountCheck.Code = "EXCHANGE_STATE_UNKNOWN"
accountCheck.Message = "Could not verify the exchange account right now."
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
}
if state.Status != exchangeAccountStatusOK {
accountCheck.Status = launchCheckStatusFailed
accountCheck.Code = state.ErrorCode
accountCheck.Message = state.ErrorMessage
if accountCheck.Message == "" {
accountCheck.Message = fmt.Sprintf("Exchange account \"%s\" is not ready (%s).", exchangeDisplayName(exchange), state.Status)
}
fundsCheck.Status = launchCheckStatusSkipped
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
}
accountCheck.Status = launchCheckStatusOK
accountCheck.Message = state.DisplayBalance
// Gate on the better of available balance and total equity: a bot with
// capital deployed in open positions has low *available* margin but is
// clearly funded — restarting it must not be blocked.
funded := state.AvailableBalance
if state.TotalEquity > funded {
funded = state.TotalEquity
}
fundsCheck.Actual = &funded
if funded < MinTradingUSDC {
message := fmt.Sprintf(
"Exchange account \"%s\" holds %.2f %s but needs at least %.0f %s to place the first trade.",
exchangeDisplayName(exchange), funded, fundsCheck.Asset, MinTradingUSDC, fundsCheck.Asset,
)
if exchange.Testnet {
// Testnet balances are play money — warn instead of block.
fundsCheck.Status = launchCheckStatusWarning
} else {
fundsCheck.Status = launchCheckStatusFailed
}
fundsCheck.Code = "EXCHANGE_INSUFFICIENT_FUNDS"
fundsCheck.Message = message
} else {
fundsCheck.Status = launchCheckStatusOK
}
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
}
// describeExchangeConfigIssue mirrors the create-time exchange validation but
// returns stable uppercase codes for the checklist UI.
func describeExchangeConfigIssue(exchange *store.Exchange) (string, string) {
if exchange == nil {
return "The selected exchange account was not found. Connect an exchange first.", "EXCHANGE_NOT_FOUND"
}
if !exchange.Enabled {
return fmt.Sprintf("Exchange account \"%s\" is disabled. Enable it first.", exchangeDisplayName(exchange)), "EXCHANGE_DISABLED"
}
if missing := missingExchangeFields(exchange); len(missing) > 0 {
return fmt.Sprintf(
"Exchange account \"%s\" is missing %s. Complete the connection first.",
exchangeDisplayName(exchange), strings.Join(missing, ", "),
), "EXCHANGE_MISSING_FIELDS"
}
if exchange.ExchangeType == "hyperliquid" && !exchange.HyperliquidBuilderApproved {
return "Hyperliquid builder authorization is not complete. Reconnect the wallet and finish the authorization.", "HYPERLIQUID_BUILDER_NOT_APPROVED"
}
return "", ""
}

View File

@@ -0,0 +1,293 @@
package api
import (
"errors"
"testing"
"nofx/crypto"
"nofx/store"
)
// Well-known throwaway development key (hardhat account #1) — never funded.
const testClaw402Key = "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
func withAIWalletBalance(t *testing.T, balance float64, err error) {
t.Helper()
original := queryAIWalletBalance
queryAIWalletBalance = func(string) (float64, error) {
return balance, err
}
t.Cleanup(func() {
queryAIWalletBalance = original
})
}
func findCheck(t *testing.T, checks []LaunchCheck, id string) LaunchCheck {
t.Helper()
for _, check := range checks {
if check.ID == id {
return check
}
}
t.Fatalf("check %q not found in %+v", id, checks)
return LaunchCheck{}
}
func TestCheckLaunchAIModel(t *testing.T) {
if got := checkLaunchAIModel(nil); got.Status != launchCheckStatusFailed || got.Code != "MODEL_NOT_FOUND" {
t.Fatalf("nil model: expected failed/MODEL_NOT_FOUND, got %+v", got)
}
disabled := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: false}
if got := checkLaunchAIModel(disabled); got.Code != "MODEL_DISABLED" {
t.Fatalf("disabled model: expected MODEL_DISABLED, got %+v", got)
}
noKey := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true}
if got := checkLaunchAIModel(noKey); got.Code != "MODEL_MISSING_CREDENTIALS" {
t.Fatalf("missing key: expected MODEL_MISSING_CREDENTIALS, got %+v", got)
}
ready := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
if got := checkLaunchAIModel(ready); got.Status != launchCheckStatusOK {
t.Fatalf("ready model: expected ok, got %+v", got)
}
}
func TestCheckLaunchAIWalletSkipsNonClaw402(t *testing.T) {
model := &store.AIModel{Name: "DeepSeek", Provider: "deepseek", Enabled: true, APIKey: crypto.EncryptedString("sk-test")}
checks := checkLaunchAIWallet(model)
if got := findCheck(t, checks, launchCheckAIWallet); got.Status != launchCheckStatusSkipped {
t.Fatalf("non-claw402 wallet check should be skipped, got %+v", got)
}
if got := findCheck(t, checks, launchCheckAIWalletFunds); got.Status != launchCheckStatusSkipped {
t.Fatalf("non-claw402 funds check should be skipped, got %+v", got)
}
}
func TestCheckLaunchAIWalletInvalidKey(t *testing.T) {
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString("not-a-key")}
checks := checkLaunchAIWallet(model)
got := findCheck(t, checks, launchCheckAIWallet)
if got.Status != launchCheckStatusFailed || got.Code != "AI_WALLET_INVALID_KEY" {
t.Fatalf("invalid key: expected failed/AI_WALLET_INVALID_KEY, got %+v", got)
}
if funds := findCheck(t, checks, launchCheckAIWalletFunds); funds.Status != launchCheckStatusSkipped {
t.Fatalf("funds check should be skipped when the key is invalid, got %+v", funds)
}
}
func TestCheckLaunchAIWalletInsufficientFunds(t *testing.T) {
withAIWalletBalance(t, 0.25, nil)
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
checks := checkLaunchAIWallet(model)
wallet := findCheck(t, checks, launchCheckAIWallet)
if wallet.Status != launchCheckStatusOK || wallet.Address == "" {
t.Fatalf("wallet check should pass with derived address, got %+v", wallet)
}
funds := findCheck(t, checks, launchCheckAIWalletFunds)
if funds.Status != launchCheckStatusFailed || funds.Code != "AI_WALLET_INSUFFICIENT_FUNDS" {
t.Fatalf("expected failed/AI_WALLET_INSUFFICIENT_FUNDS, got %+v", funds)
}
if funds.Actual == nil || *funds.Actual != 0.25 {
t.Fatalf("expected actual balance 0.25, got %+v", funds.Actual)
}
if funds.Required != MinAIFeeUSDC {
t.Fatalf("expected required %v, got %v", MinAIFeeUSDC, funds.Required)
}
if funds.Address == "" {
t.Fatalf("funds check should carry the deposit address")
}
}
func TestCheckLaunchAIWalletRPCOutageIsWarningNotFailure(t *testing.T) {
withAIWalletBalance(t, 0, errors.New("rpc unreachable"))
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
checks := checkLaunchAIWallet(model)
funds := findCheck(t, checks, launchCheckAIWalletFunds)
if funds.Status != launchCheckStatusWarning || funds.Code != "AI_WALLET_BALANCE_UNKNOWN" {
t.Fatalf("RPC outage must degrade to warning, got %+v", funds)
}
}
func TestCheckLaunchAIWalletFunded(t *testing.T) {
withAIWalletBalance(t, 25.5, nil)
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
checks := checkLaunchAIWallet(model)
funds := findCheck(t, checks, launchCheckAIWalletFunds)
if funds.Status != launchCheckStatusOK {
t.Fatalf("funded wallet should pass, got %+v", funds)
}
}
func TestDescribeExchangeConfigIssue(t *testing.T) {
if _, code := describeExchangeConfigIssue(nil); code != "EXCHANGE_NOT_FOUND" {
t.Fatalf("nil exchange: expected EXCHANGE_NOT_FOUND, got %s", code)
}
disabled := &store.Exchange{ID: "ex", ExchangeType: "hyperliquid", Enabled: false}
if _, code := describeExchangeConfigIssue(disabled); code != "EXCHANGE_DISABLED" {
t.Fatalf("disabled: expected EXCHANGE_DISABLED, got %s", code)
}
missing := &store.Exchange{ID: "ex", ExchangeType: "hyperliquid", Enabled: true}
if _, code := describeExchangeConfigIssue(missing); code != "EXCHANGE_MISSING_FIELDS" {
t.Fatalf("missing fields: expected EXCHANGE_MISSING_FIELDS, got %s", code)
}
unapproved := &store.Exchange{
ID: "ex",
ExchangeType: "hyperliquid",
Enabled: true,
APIKey: crypto.EncryptedString(testClaw402Key),
HyperliquidWalletAddr: "0x1111111111111111111111111111111111111111",
}
if _, code := describeExchangeConfigIssue(unapproved); code != "HYPERLIQUID_BUILDER_NOT_APPROVED" {
t.Fatalf("builder unapproved: expected HYPERLIQUID_BUILDER_NOT_APPROVED, got %s", code)
}
unapproved.HyperliquidBuilderApproved = true
if _, code := describeExchangeConfigIssue(unapproved); code != "" {
t.Fatalf("ready exchange: expected no issue, got %s", code)
}
}
func readyHyperliquidExchange() *store.Exchange {
return &store.Exchange{
ID: "ex-hl",
ExchangeType: "hyperliquid",
Enabled: true,
APIKey: crypto.EncryptedString(testClaw402Key),
HyperliquidWalletAddr: "0x1111111111111111111111111111111111111111",
HyperliquidBuilderApproved: true,
}
}
func preflightTestServer(t *testing.T, userID string, states map[string]ExchangeAccountState) *Server {
t.Helper()
server := &Server{exchangeAccountStateCache: NewExchangeAccountStateCache()}
if states != nil {
server.exchangeAccountStateCache.Set(userID, states)
}
return server
}
func TestCheckLaunchExchangeInsufficientFundsBlocks(t *testing.T) {
exchange := readyHyperliquidExchange()
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 5.5, TotalEquity: 5.5},
})
checks := server.checkLaunchExchange("user-1", exchange)
funds := findCheck(t, checks, launchCheckExchangeFunds)
if funds.Status != launchCheckStatusFailed || funds.Code != "EXCHANGE_INSUFFICIENT_FUNDS" {
t.Fatalf("expected failed/EXCHANGE_INSUFFICIENT_FUNDS, got %+v", funds)
}
if funds.Actual == nil || *funds.Actual != 5.5 {
t.Fatalf("expected actual 5.5, got %+v", funds.Actual)
}
}
func TestCheckLaunchExchangeDeployedMarginPasses(t *testing.T) {
// A running bot with capital locked in positions: low available balance
// but healthy equity. Restart must not be blocked.
exchange := readyHyperliquidExchange()
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 3, TotalEquity: 100},
})
checks := server.checkLaunchExchange("user-1", exchange)
funds := findCheck(t, checks, launchCheckExchangeFunds)
if funds.Status != launchCheckStatusOK {
t.Fatalf("deployed margin with healthy equity should pass, got %+v", funds)
}
if funds.Actual == nil || *funds.Actual != 100 {
t.Fatalf("expected actual 100 (equity), got %+v", funds.Actual)
}
}
func TestCheckLaunchExchangeTestnetLowFundsIsWarning(t *testing.T) {
exchange := readyHyperliquidExchange()
exchange.Testnet = true
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 0},
})
checks := server.checkLaunchExchange("user-1", exchange)
funds := findCheck(t, checks, launchCheckExchangeFunds)
if funds.Status != launchCheckStatusWarning {
t.Fatalf("testnet low funds should warn, not block, got %+v", funds)
}
}
func TestCheckLaunchExchangeInvalidCredentials(t *testing.T) {
exchange := readyHyperliquidExchange()
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {
ExchangeID: exchange.ID,
Status: exchangeAccountStatusInvalidCredentials,
ErrorCode: "INVALID_CREDENTIALS",
ErrorMessage: "Exchange credentials are invalid",
},
})
checks := server.checkLaunchExchange("user-1", exchange)
account := findCheck(t, checks, launchCheckExchangeAccount)
if account.Status != launchCheckStatusFailed || account.Code != "INVALID_CREDENTIALS" {
t.Fatalf("expected failed/INVALID_CREDENTIALS, got %+v", account)
}
if funds := findCheck(t, checks, launchCheckExchangeFunds); funds.Status != launchCheckStatusSkipped {
t.Fatalf("funds check should be skipped when account probe fails, got %+v", funds)
}
}
func TestRunLaunchPreflightAggregatesReadiness(t *testing.T) {
withAIWalletBalance(t, 10, nil)
exchange := readyHyperliquidExchange()
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 100, TotalEquity: 100},
})
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
strategy := &store.Strategy{ID: "strat-1", Name: "Autopilot"}
result := server.runLaunchPreflight("user-1", model, exchange, strategy, true)
if !result.Ready {
t.Fatalf("expected ready, got %+v", result)
}
if result.MinAIFeeUSDC != MinAIFeeUSDC || result.MinTradingUSDC != MinTradingUSDC {
t.Fatalf("minimums must be exposed in the response, got %+v", result)
}
// Break one prerequisite → not ready, and Summary explains it.
withAIWalletBalance(t, 0, nil)
result = server.runLaunchPreflight("user-1", model, exchange, strategy, true)
if result.Ready {
t.Fatalf("expected not ready with empty AI wallet, got %+v", result)
}
if result.Summary() == "" {
t.Fatalf("summary should describe the failing check")
}
}
func TestRunLaunchPreflightWarningsDoNotBlock(t *testing.T) {
withAIWalletBalance(t, 0, errors.New("rpc down"))
exchange := readyHyperliquidExchange()
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 100},
})
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
result := server.runLaunchPreflight("user-1", model, exchange, nil, false)
if !result.Ready {
t.Fatalf("warnings must not block launch, got %+v", result)
}
}

View File

@@ -237,8 +237,17 @@ Only include fields you want to change.`,
`:id = trader_id from GET /api/my-traders. Stops and permanently removes the trader and all its data.`,
s.handleDeleteTrader)
s.routeWithSchema(protected, "POST", "/traders/:id/start", "Start trader — begins live trading",
`:id = trader_id from GET /api/my-traders. No request body needed. The trader must have a valid exchange and AI model configured.`,
`:id = trader_id from GET /api/my-traders. No request body needed. The trader must have a valid exchange and AI model configured.
Runs launch preflight first and returns 400 with {"error_key":"trader.start.preflight_failed","preflight":{...}} when checks fail. Append ?force=true to skip balance gates.`,
s.handleStartTrader)
s.routeWithSchema(protected, "GET", "/traders/:id/preflight", "Run launch readiness checks for a trader",
`:id = trader_id from GET /api/my-traders.
Returns: {"ready":<bool>,"checks":[{"id":"ai_model|ai_wallet|ai_wallet_funds|strategy|exchange_config|exchange_account|exchange_funds","status":"ok|failed|warning|skipped","code":"<string>","message":"<string>","required":<number>,"actual":<number>,"asset":"<string>","address":"<string>"}],"min_ai_fee_usdc":<number>,"min_trading_usdc":<number>}`,
s.handleTraderPreflight)
s.routeWithSchema(protected, "POST", "/launch/preflight", "Run launch readiness checks before creating a trader",
`Body: {"ai_model_id":"<EXACT id from GET /api/models>","exchange_id":"<EXACT id from GET /api/exchanges>","strategy_id":"<optional, EXACT id from GET /api/strategies>"}
Returns the same shape as GET /api/traders/:id/preflight. Use this before POST /api/traders to surface every missing prerequisite at once.`,
s.handleLaunchPreflight)
s.routeWithSchema(protected, "POST", "/traders/:id/stop", "Stop trader — halts live trading",
`:id = trader_id from GET /api/my-traders. No request body needed. Gracefully stops the trading loop.`,
s.handleStopTrader)

View File

@@ -177,8 +177,12 @@ type AutoTrader struct {
gridState *GridState // Grid trading state (only used when StrategyType == "grid_trading")
claw402WalletAddr string // Claw402 wallet address (derived from private key at start)
consecutiveAIFailures int // Consecutive AI call failures
runtimeHealthMu sync.RWMutex // Guards safe mode + AI wallet health (loop writes, API reads)
safeMode bool // Safe mode: no new positions, protect existing ones
safeModeReason string // Why safe mode was activated
aiWalletStatus string // "ok"|"low"|"empty"|"unknown" — see runtime_health.go
aiWalletBalanceUSDC float64 // Last observed Base USDC balance of the claw402 wallet
aiWalletCheckedAt time.Time // When the balance was last observed
}
// NewAutoTrader creates an automatic trader
@@ -710,7 +714,9 @@ func (at *AutoTrader) runPreLaunchChecks() {
balance, err := wallet.QueryUSDCBalance(addr)
if err != nil {
logger.Warnf("⚠️ [%s] Could not query USDC balance: %v", at.name, err)
at.markAIWalletHealthUnknown()
} else {
at.setAIWalletHealth(balance)
// Estimate runway
scanMinutes := int(at.config.ScanInterval.Minutes())
modelName := at.config.CustomModelName

View File

@@ -91,6 +91,19 @@ func (at *AutoTrader) GetStatus() map[string]interface{} {
}
}
// Runtime health: safe mode + AI fee wallet, so the dashboard can show a
// persistent banner instead of the user digging through logs.
safeMode, safeModeReason := at.safeModeState()
result["safe_mode"] = safeMode
if safeModeReason != "" {
result["safe_mode_reason"] = safeModeReason
}
if status, balance, checkedAt := at.aiWalletHealth(); status != "" {
result["ai_wallet_status"] = status
result["ai_wallet_balance_usdc"] = balance
result["ai_wallet_checked_at"] = checkedAt.Format(time.RFC3339)
}
return result
}

View File

@@ -21,7 +21,7 @@ import (
// - never exceeds MaxPositions,
// - never doubles a base symbol already held or already in the decision set.
func (at *AutoTrader) ensureLongShortCoverage(decisions []kernel.Decision, ctx *kernel.Context, equity float64) []kernel.Decision {
if at == nil || ctx == nil || at.safeMode {
if at == nil || ctx == nil || at.isSafeMode() {
return decisions
}
if at.config.StrategyConfig == nil || at.config.StrategyConfig.CoinSource.SourceType != "vergex_signal" {

View File

@@ -2,10 +2,12 @@ package trader
import (
"encoding/json"
"errors"
"fmt"
"nofx/kernel"
"nofx/logger"
"nofx/market"
"nofx/mcp/payment"
"nofx/provider/hyperliquid"
"nofx/store"
"nofx/wallet"
@@ -141,10 +143,21 @@ func (at *AutoTrader) runCycle() error {
record.Success = false
record.ErrorMessage = fmt.Sprintf("Failed to get AI decision: %v", err)
// Payment-layer rejection means the AI fee wallet is definitively out
// of funds — surface it as structured health state (GetStatus), not
// just a log line.
var insufficientFunds *payment.ErrInsufficientFunds
if errors.As(err, &insufficientFunds) {
at.markAIWalletEmptyFromPayment(insufficientFunds.Balance)
record.ErrorMessage = fmt.Sprintf(
"AI fee wallet out of funds: balance $%.2f USDC, next call needs ~$%.2f. Top up the Base USDC wallet.",
insufficientFunds.Balance, insufficientFunds.Needed,
)
}
// Activate safe mode after 3 consecutive failures
if at.consecutiveAIFailures >= 3 && !at.safeMode {
at.safeMode = true
at.safeModeReason = fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err)
if at.consecutiveAIFailures >= 3 && !at.isSafeMode() {
at.setSafeMode(true, fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err))
at.logErrorf("🛡️ SAFE MODE ACTIVATED — AI failed %d times in a row. No new positions will be opened. Existing positions are protected with current stop-loss settings.", at.consecutiveAIFailures)
at.logErrorf("🛡️ Reason: %v", err)
at.logErrorf("🛡️ Action: Will keep trying AI each cycle. Safe mode auto-deactivates when AI recovers.")
@@ -172,7 +185,7 @@ func (at *AutoTrader) runCycle() error {
}
// In safe mode, don't return error — keep the loop running to retry next cycle
if at.safeMode {
if at.isSafeMode() {
at.logWarnf("🛡️ Safe mode: skipping this cycle, will retry in %v", at.config.ScanInterval)
return nil
}
@@ -185,10 +198,9 @@ func (at *AutoTrader) runCycle() error {
at.logInfof("✅ AI recovered after %d consecutive failures", at.consecutiveAIFailures)
}
at.consecutiveAIFailures = 0
if at.safeMode {
if at.isSafeMode() {
at.logInfof("🛡️ SAFE MODE DEACTIVATED — AI is working again. Resuming normal trading.")
at.safeMode = false
at.safeModeReason = ""
at.setSafeMode(false, "")
}
// // 5. Print system prompt
@@ -242,7 +254,7 @@ func (at *AutoTrader) runCycle() error {
}
// Safe mode: filter out open positions, only allow close/hold
if at.safeMode {
if at.isSafeMode() {
filtered := make([]kernel.Decision, 0)
for _, d := range sortedDecisions {
if d.Action == "open_long" || d.Action == "open_short" {
@@ -770,10 +782,12 @@ func (at *AutoTrader) checkClaw402Balance() {
balance, err := wallet.QueryUSDCBalance(at.claw402WalletAddr)
if err != nil {
at.logWarnf("⚠️ Failed to query USDC balance: %v", err)
at.markAIWalletHealthUnknown()
return
}
if balance < 1.0 {
at.setAIWalletHealth(balance)
if balance < aiWalletLowThresholdUSDC {
at.logWarnf("⚠️ Low USDC balance: $%.2f — AI may stop soon!", balance)
}
if balance <= 0 {

82
trader/runtime_health.go Normal file
View File

@@ -0,0 +1,82 @@
package trader
import (
"time"
)
// Runtime health state written by the run-loop goroutine and read by the API
// layer (GetStatus). Everything here goes through runtimeHealthMu so the
// dashboard can poll without racing the trading loop.
// AI fee wallet health statuses exposed via GetStatus as "ai_wallet_status".
const (
AIWalletStatusOK = "ok"
AIWalletStatusLow = "low"
AIWalletStatusEmpty = "empty"
AIWalletStatusUnknown = "unknown"
)
// aiWalletLowThresholdUSDC mirrors api.MinAIFeeUSDC: below this the wallet
// cannot reliably pay for the next AI/data calls.
const aiWalletLowThresholdUSDC = 1.0
func (at *AutoTrader) setSafeMode(active bool, reason string) {
at.runtimeHealthMu.Lock()
at.safeMode = active
at.safeModeReason = reason
at.runtimeHealthMu.Unlock()
}
func (at *AutoTrader) isSafeMode() bool {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.safeMode
}
func (at *AutoTrader) safeModeState() (bool, string) {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.safeMode, at.safeModeReason
}
// setAIWalletHealth records a fresh balance observation for the claw402 wallet.
func (at *AutoTrader) setAIWalletHealth(balance float64) {
status := AIWalletStatusOK
switch {
case balance <= 0:
status = AIWalletStatusEmpty
case balance < aiWalletLowThresholdUSDC:
status = AIWalletStatusLow
}
at.runtimeHealthMu.Lock()
at.aiWalletStatus = status
at.aiWalletBalanceUSDC = balance
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
// markAIWalletHealthUnknown keeps the last observed balance but flags that the
// current reading could not be refreshed (e.g. Base RPC unreachable).
func (at *AutoTrader) markAIWalletHealthUnknown() {
at.runtimeHealthMu.Lock()
at.aiWalletStatus = AIWalletStatusUnknown
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
// markAIWalletEmptyFromPayment records a payment-layer rejection: the wallet
// definitively could not cover an AI call.
func (at *AutoTrader) markAIWalletEmptyFromPayment(balance float64) {
at.runtimeHealthMu.Lock()
at.aiWalletStatus = AIWalletStatusEmpty
at.aiWalletBalanceUSDC = balance
at.aiWalletCheckedAt = time.Now().UTC()
at.runtimeHealthMu.Unlock()
}
func (at *AutoTrader) aiWalletHealth() (status string, balance float64, checkedAt time.Time) {
at.runtimeHealthMu.RLock()
defer at.runtimeHealthMu.RUnlock()
return at.aiWalletStatus, at.aiWalletBalanceUSDC, at.aiWalletCheckedAt
}