mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-06 04:20:59 +08:00
feat: server-side launch preflight with structured checks and runtime AI wallet health
- POST /api/launch/preflight and GET /api/traders/:id/preflight run every launch prerequisite (AI model credential, claw402 wallet key + Base USDC balance, strategy, exchange config, live account state, funding minimums) and return a structured checklist with stable codes; minimums (1 USDC AI fee / 12 USDC trading) now live server-side and are exposed in the response. - POST /traders/:id/start enforces the preflight (400 with error_key trader.start.preflight_failed + full checklist; ?force=true to override). Funding gates use max(available, equity) so restarting a bot with deployed margin is not blocked; indeterminate probes (RPC outage) degrade to warnings instead of blocking. - Run loop now surfaces runtime health via GetStatus: safe_mode(+reason) and ai_wallet_status/balance (ok|low|empty|unknown), including typed detection of claw402 ErrInsufficientFunds instead of burying it in logs; safe-mode state is mutex-guarded for API readers. - Onboarding wallet endpoints switch to the cached, error-aware balance query and report balance_status so the UI can tell an RPC outage from an empty wallet.
This commit is contained in:
@@ -26,6 +26,7 @@ type beginnerOnboardingResponse struct {
|
||||
DefaultModel string `json:"default_model"`
|
||||
ConfiguredModelID string `json:"configured_model_id"`
|
||||
BalanceUSDC string `json:"balance_usdc"`
|
||||
BalanceStatus string `json:"balance_status,omitempty"`
|
||||
EnvSaved bool `json:"env_saved"`
|
||||
EnvPath string `json:"env_path,omitempty"`
|
||||
ReusedExisting bool `json:"reused_existing"`
|
||||
@@ -36,10 +37,22 @@ type currentBeginnerWalletResponse struct {
|
||||
Found bool `json:"found"`
|
||||
Address string `json:"address,omitempty"`
|
||||
BalanceUSDC string `json:"balance_usdc,omitempty"`
|
||||
BalanceStatus string `json:"balance_status,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Claw402Status string `json:"claw402_status"`
|
||||
}
|
||||
|
||||
// queryBeginnerWalletBalance returns the wallet balance plus a status flag so
|
||||
// the UI can distinguish "RPC unreachable" from a genuinely empty wallet.
|
||||
// Uses the 30s cache — safe for UI polling.
|
||||
func queryBeginnerWalletBalance(address string) (balanceUSDC string, balanceStatus string) {
|
||||
balance, err := wallet.QueryUSDCBalanceCached(address)
|
||||
if err != nil {
|
||||
return "", "unknown"
|
||||
}
|
||||
return fmt.Sprintf("%.2f", balance), "ok"
|
||||
}
|
||||
|
||||
func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
if userID == "" {
|
||||
@@ -72,6 +85,7 @@ func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
|
||||
os.Setenv("CLAW402_DEFAULT_MODEL", payment.DefaultClaw402Model)
|
||||
|
||||
envSaved, envPath, envErr := persistBeginnerWalletEnv(privateKey, address)
|
||||
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
|
||||
resp := beginnerOnboardingResponse{
|
||||
Address: address,
|
||||
PrivateKey: privateKey,
|
||||
@@ -80,7 +94,8 @@ func (s *Server) handleBeginnerOnboarding(c *gin.Context) {
|
||||
Provider: "claw402",
|
||||
DefaultModel: payment.DefaultClaw402Model,
|
||||
ConfiguredModelID: configuredModelID,
|
||||
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
|
||||
BalanceUSDC: balanceUSDC,
|
||||
BalanceStatus: balanceStatus,
|
||||
EnvSaved: envSaved,
|
||||
EnvPath: envPath,
|
||||
ReusedExisting: reusedExisting,
|
||||
@@ -124,10 +139,12 @@ func (s *Server) handleCurrentBeginnerWallet(c *gin.Context) {
|
||||
continue
|
||||
}
|
||||
|
||||
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
|
||||
c.JSON(http.StatusOK, currentBeginnerWalletResponse{
|
||||
Found: true,
|
||||
Address: address,
|
||||
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
|
||||
BalanceUSDC: balanceUSDC,
|
||||
BalanceStatus: balanceStatus,
|
||||
Source: "model",
|
||||
Claw402Status: claw402Status,
|
||||
})
|
||||
@@ -136,10 +153,12 @@ func (s *Server) handleCurrentBeginnerWallet(c *gin.Context) {
|
||||
|
||||
address := strings.TrimSpace(os.Getenv("CLAW402_WALLET_ADDRESS"))
|
||||
if address != "" {
|
||||
balanceUSDC, balanceStatus := queryBeginnerWalletBalance(address)
|
||||
c.JSON(http.StatusOK, currentBeginnerWalletResponse{
|
||||
Found: true,
|
||||
Address: address,
|
||||
BalanceUSDC: wallet.QueryUSDCBalanceStr(address),
|
||||
BalanceUSDC: balanceUSDC,
|
||||
BalanceStatus: balanceStatus,
|
||||
Source: "env",
|
||||
Claw402Status: claw402Status,
|
||||
})
|
||||
|
||||
@@ -850,6 +850,24 @@ func (s *Server) handleStartTrader(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Server-side launch gate: the trader cannot function without a funded AI
|
||||
// wallet and a ready exchange account, so verify both before the run loop
|
||||
// starts. `?force=true` skips the gate for deliberate manual overrides.
|
||||
if c.Query("force") != "true" {
|
||||
// strategyRequired=false: a trader that loaded into memory necessarily
|
||||
// has a valid strategy (the manager refuses to load without one), so the
|
||||
// preflight strategy check would be redundant here.
|
||||
preflight := s.runLaunchPreflight(userID, fullCfg.AIModel, fullCfg.Exchange, fullCfg.Strategy, false)
|
||||
if !preflight.Ready {
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": formatTraderStartError(preflight.Summary(), "Complete the failing checks, then start the bot again"),
|
||||
"error_key": "trader.start.preflight_failed",
|
||||
"preflight": preflight,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Start trader
|
||||
go func() {
|
||||
logger.Infof("▶️ Starting trader %s (%s)", traderID, trader.GetName())
|
||||
|
||||
375
api/launch_preflight.go
Normal file
375
api/launch_preflight.go
Normal file
@@ -0,0 +1,375 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nofx/store"
|
||||
"nofx/wallet"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Launch readiness minimums. These are the single source of truth — the
|
||||
// frontend reads them from the preflight response instead of hardcoding.
|
||||
const (
|
||||
// MinAIFeeUSDC is the minimum Base USDC a claw402 fee wallet needs so the
|
||||
// trader can pay for its first AI/data calls.
|
||||
MinAIFeeUSDC = 1.0
|
||||
// MinTradingUSDC is the minimum available balance the exchange account
|
||||
// needs before the trader can place its first order.
|
||||
MinTradingUSDC = 12.0
|
||||
)
|
||||
|
||||
const (
|
||||
launchCheckStatusOK = "ok"
|
||||
launchCheckStatusFailed = "failed"
|
||||
launchCheckStatusWarning = "warning"
|
||||
launchCheckStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
// Check IDs — stable identifiers the frontend maps to guided-setup steps.
|
||||
const (
|
||||
launchCheckAIModel = "ai_model"
|
||||
launchCheckAIWallet = "ai_wallet"
|
||||
launchCheckAIWalletFunds = "ai_wallet_funds"
|
||||
launchCheckStrategy = "strategy"
|
||||
launchCheckExchangeConfig = "exchange_config"
|
||||
launchCheckExchangeAccount = "exchange_account"
|
||||
launchCheckExchangeFunds = "exchange_funds"
|
||||
)
|
||||
|
||||
// queryAIWalletBalance is swappable in tests to avoid live Base RPC calls.
|
||||
var queryAIWalletBalance = wallet.QueryUSDCBalanceCached
|
||||
|
||||
type LaunchCheck struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
// Numeric context so the UI can render progress like "8.40 / 12 USDC".
|
||||
Required float64 `json:"required,omitempty"`
|
||||
Actual *float64 `json:"actual,omitempty"`
|
||||
Asset string `json:"asset,omitempty"`
|
||||
// Address is the funding address for balance checks, so the UI can offer
|
||||
// a deposit shortcut next to the failing item.
|
||||
Address string `json:"address,omitempty"`
|
||||
}
|
||||
|
||||
type LaunchPreflightResult struct {
|
||||
Ready bool `json:"ready"`
|
||||
Checks []LaunchCheck `json:"checks"`
|
||||
MinAIFeeUSDC float64 `json:"min_ai_fee_usdc"`
|
||||
MinTradingUSDC float64 `json:"min_trading_usdc"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
}
|
||||
|
||||
func (r LaunchPreflightResult) failedChecks() []LaunchCheck {
|
||||
var failed []LaunchCheck
|
||||
for _, check := range r.Checks {
|
||||
if check.Status == launchCheckStatusFailed {
|
||||
failed = append(failed, check)
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
// Summary joins the failing messages into one human-readable sentence.
|
||||
func (r LaunchPreflightResult) Summary() string {
|
||||
failed := r.failedChecks()
|
||||
if len(failed) == 0 {
|
||||
return ""
|
||||
}
|
||||
messages := make([]string, 0, len(failed))
|
||||
for _, check := range failed {
|
||||
if check.Message != "" {
|
||||
messages = append(messages, check.Message)
|
||||
}
|
||||
}
|
||||
return strings.Join(messages, " ")
|
||||
}
|
||||
|
||||
type launchPreflightRequest struct {
|
||||
AIModelID string `json:"ai_model_id"`
|
||||
ExchangeID string `json:"exchange_id"`
|
||||
StrategyID string `json:"strategy_id"`
|
||||
}
|
||||
|
||||
// handleLaunchPreflight runs launch readiness checks for a model/exchange
|
||||
// pair before any trader is created or mutated. POST /api/launch/preflight
|
||||
func (s *Server) handleLaunchPreflight(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
|
||||
var req launchPreflightRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
SafeBadRequest(c, "Invalid preflight request")
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.AIModelID) == "" || strings.TrimSpace(req.ExchangeID) == "" {
|
||||
SafeBadRequest(c, "ai_model_id and exchange_id are required")
|
||||
return
|
||||
}
|
||||
|
||||
model, err := s.store.AIModel().Get(userID, req.AIModelID)
|
||||
if err != nil {
|
||||
model = nil
|
||||
}
|
||||
exchange, err := s.store.Exchange().GetByID(userID, req.ExchangeID)
|
||||
if err != nil {
|
||||
exchange = nil
|
||||
}
|
||||
|
||||
var strategy *store.Strategy
|
||||
strategyRequired := strings.TrimSpace(req.StrategyID) != ""
|
||||
if strategyRequired {
|
||||
strategy, err = s.store.Strategy().Get(userID, req.StrategyID)
|
||||
if err != nil {
|
||||
strategy = nil
|
||||
}
|
||||
}
|
||||
|
||||
result := s.runLaunchPreflight(userID, model, exchange, strategy, strategyRequired)
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// handleTraderPreflight runs the same checks against an existing trader's
|
||||
// full configuration. GET /api/traders/:id/preflight
|
||||
func (s *Server) handleTraderPreflight(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
traderID := c.Param("id")
|
||||
|
||||
fullCfg, err := s.store.Trader().GetFullConfig(userID, traderID)
|
||||
if err != nil || fullCfg == nil || fullCfg.Trader == nil {
|
||||
SafeNotFound(c, "Trader")
|
||||
return
|
||||
}
|
||||
|
||||
result := s.runLaunchPreflight(userID, fullCfg.AIModel, fullCfg.Exchange, fullCfg.Strategy, true)
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// runLaunchPreflight composes every launch readiness check. A check that
|
||||
// cannot be evaluated (RPC outage, probe timeout) reports "warning" instead
|
||||
// of "failed" so an infrastructure hiccup never hard-blocks a launch.
|
||||
func (s *Server) runLaunchPreflight(
|
||||
userID string,
|
||||
model *store.AIModel,
|
||||
exchange *store.Exchange,
|
||||
strategy *store.Strategy,
|
||||
strategyRequired bool,
|
||||
) LaunchPreflightResult {
|
||||
checks := []LaunchCheck{checkLaunchAIModel(model)}
|
||||
checks = append(checks, checkLaunchAIWallet(model)...)
|
||||
checks = append(checks, checkLaunchStrategy(strategy, strategyRequired))
|
||||
checks = append(checks, s.checkLaunchExchange(userID, exchange)...)
|
||||
|
||||
ready := true
|
||||
for _, check := range checks {
|
||||
if check.Status == launchCheckStatusFailed {
|
||||
ready = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return LaunchPreflightResult{
|
||||
Ready: ready,
|
||||
Checks: checks,
|
||||
MinAIFeeUSDC: MinAIFeeUSDC,
|
||||
MinTradingUSDC: MinTradingUSDC,
|
||||
CheckedAt: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
|
||||
func checkLaunchAIModel(model *store.AIModel) LaunchCheck {
|
||||
check := LaunchCheck{ID: launchCheckAIModel}
|
||||
|
||||
switch {
|
||||
case model == nil:
|
||||
check.Status = launchCheckStatusFailed
|
||||
check.Code = "MODEL_NOT_FOUND"
|
||||
check.Message = "The selected AI model was not found. Configure an AI model first."
|
||||
case !model.Enabled:
|
||||
check.Status = launchCheckStatusFailed
|
||||
check.Code = "MODEL_DISABLED"
|
||||
check.Message = fmt.Sprintf("AI model \"%s\" is disabled. Enable it first.", model.Name)
|
||||
case strings.TrimSpace(model.APIKey.String()) == "":
|
||||
check.Status = launchCheckStatusFailed
|
||||
check.Code = "MODEL_MISSING_CREDENTIALS"
|
||||
check.Message = fmt.Sprintf("AI model \"%s\" has no credential saved. Add the API key or wallet key first.", model.Name)
|
||||
default:
|
||||
check.Status = launchCheckStatusOK
|
||||
check.Message = model.Name
|
||||
}
|
||||
|
||||
return check
|
||||
}
|
||||
|
||||
// checkLaunchAIWallet validates the claw402 fee wallet (address + Base USDC
|
||||
// balance). Non-claw402 providers pay per API key, so both checks are skipped.
|
||||
func checkLaunchAIWallet(model *store.AIModel) []LaunchCheck {
|
||||
walletCheck := LaunchCheck{ID: launchCheckAIWallet}
|
||||
fundsCheck := LaunchCheck{ID: launchCheckAIWalletFunds, Asset: "USDC", Required: MinAIFeeUSDC}
|
||||
|
||||
if model == nil || model.Provider != "claw402" || strings.TrimSpace(model.APIKey.String()) == "" {
|
||||
walletCheck.Status = launchCheckStatusSkipped
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{walletCheck, fundsCheck}
|
||||
}
|
||||
|
||||
address, err := walletAddressFromPrivateKey(model.APIKey.String())
|
||||
if err != nil {
|
||||
walletCheck.Status = launchCheckStatusFailed
|
||||
walletCheck.Code = "AI_WALLET_INVALID_KEY"
|
||||
walletCheck.Message = "The Claw402 wallet key is invalid. Recreate the Base USDC payment wallet."
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{walletCheck, fundsCheck}
|
||||
}
|
||||
|
||||
walletCheck.Status = launchCheckStatusOK
|
||||
walletCheck.Address = address
|
||||
fundsCheck.Address = address
|
||||
|
||||
balance, err := queryAIWalletBalance(address)
|
||||
if err != nil {
|
||||
fundsCheck.Status = launchCheckStatusWarning
|
||||
fundsCheck.Code = "AI_WALLET_BALANCE_UNKNOWN"
|
||||
fundsCheck.Message = "Could not verify the Base USDC balance right now. The trader will start, but AI calls fail if the wallet is empty."
|
||||
return []LaunchCheck{walletCheck, fundsCheck}
|
||||
}
|
||||
|
||||
fundsCheck.Actual = &balance
|
||||
if balance < MinAIFeeUSDC {
|
||||
fundsCheck.Status = launchCheckStatusFailed
|
||||
fundsCheck.Code = "AI_WALLET_INSUFFICIENT_FUNDS"
|
||||
fundsCheck.Message = fmt.Sprintf(
|
||||
"The Claw402 wallet holds %.2f USDC but needs at least %.0f USDC on Base to pay for AI and data calls.",
|
||||
balance, MinAIFeeUSDC,
|
||||
)
|
||||
} else {
|
||||
fundsCheck.Status = launchCheckStatusOK
|
||||
}
|
||||
|
||||
return []LaunchCheck{walletCheck, fundsCheck}
|
||||
}
|
||||
|
||||
func checkLaunchStrategy(strategy *store.Strategy, required bool) LaunchCheck {
|
||||
check := LaunchCheck{ID: launchCheckStrategy}
|
||||
|
||||
switch {
|
||||
case strategy != nil:
|
||||
check.Status = launchCheckStatusOK
|
||||
check.Message = strategy.Name
|
||||
case required:
|
||||
check.Status = launchCheckStatusFailed
|
||||
check.Code = "STRATEGY_NOT_FOUND"
|
||||
check.Message = "The selected strategy was not found. Pick or create a strategy first."
|
||||
default:
|
||||
check.Status = launchCheckStatusSkipped
|
||||
}
|
||||
|
||||
return check
|
||||
}
|
||||
|
||||
// checkLaunchExchange validates exchange configuration completeness and then
|
||||
// probes the live account (30s server cache) for status and balance.
|
||||
func (s *Server) checkLaunchExchange(userID string, exchange *store.Exchange) []LaunchCheck {
|
||||
configCheck := LaunchCheck{ID: launchCheckExchangeConfig}
|
||||
accountCheck := LaunchCheck{ID: launchCheckExchangeAccount}
|
||||
fundsCheck := LaunchCheck{ID: launchCheckExchangeFunds, Required: MinTradingUSDC}
|
||||
|
||||
if msg, code := describeExchangeConfigIssue(exchange); code != "" {
|
||||
configCheck.Status = launchCheckStatusFailed
|
||||
configCheck.Code = code
|
||||
configCheck.Message = msg
|
||||
accountCheck.Status = launchCheckStatusSkipped
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
|
||||
}
|
||||
|
||||
configCheck.Status = launchCheckStatusOK
|
||||
configCheck.Message = exchangeDisplayName(exchange)
|
||||
fundsCheck.Asset = accountAssetForExchange(exchange.ExchangeType)
|
||||
|
||||
states, err := s.getExchangeAccountStates(userID)
|
||||
if err != nil {
|
||||
accountCheck.Status = launchCheckStatusWarning
|
||||
accountCheck.Code = "EXCHANGE_STATE_UNKNOWN"
|
||||
accountCheck.Message = "Could not verify the exchange account right now."
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
|
||||
}
|
||||
|
||||
state, ok := states[exchange.ID]
|
||||
if !ok {
|
||||
accountCheck.Status = launchCheckStatusWarning
|
||||
accountCheck.Code = "EXCHANGE_STATE_UNKNOWN"
|
||||
accountCheck.Message = "Could not verify the exchange account right now."
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
|
||||
}
|
||||
|
||||
if state.Status != exchangeAccountStatusOK {
|
||||
accountCheck.Status = launchCheckStatusFailed
|
||||
accountCheck.Code = state.ErrorCode
|
||||
accountCheck.Message = state.ErrorMessage
|
||||
if accountCheck.Message == "" {
|
||||
accountCheck.Message = fmt.Sprintf("Exchange account \"%s\" is not ready (%s).", exchangeDisplayName(exchange), state.Status)
|
||||
}
|
||||
fundsCheck.Status = launchCheckStatusSkipped
|
||||
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
|
||||
}
|
||||
|
||||
accountCheck.Status = launchCheckStatusOK
|
||||
accountCheck.Message = state.DisplayBalance
|
||||
|
||||
// Gate on the better of available balance and total equity: a bot with
|
||||
// capital deployed in open positions has low *available* margin but is
|
||||
// clearly funded — restarting it must not be blocked.
|
||||
funded := state.AvailableBalance
|
||||
if state.TotalEquity > funded {
|
||||
funded = state.TotalEquity
|
||||
}
|
||||
fundsCheck.Actual = &funded
|
||||
|
||||
if funded < MinTradingUSDC {
|
||||
message := fmt.Sprintf(
|
||||
"Exchange account \"%s\" holds %.2f %s but needs at least %.0f %s to place the first trade.",
|
||||
exchangeDisplayName(exchange), funded, fundsCheck.Asset, MinTradingUSDC, fundsCheck.Asset,
|
||||
)
|
||||
if exchange.Testnet {
|
||||
// Testnet balances are play money — warn instead of block.
|
||||
fundsCheck.Status = launchCheckStatusWarning
|
||||
} else {
|
||||
fundsCheck.Status = launchCheckStatusFailed
|
||||
}
|
||||
fundsCheck.Code = "EXCHANGE_INSUFFICIENT_FUNDS"
|
||||
fundsCheck.Message = message
|
||||
} else {
|
||||
fundsCheck.Status = launchCheckStatusOK
|
||||
}
|
||||
|
||||
return []LaunchCheck{configCheck, accountCheck, fundsCheck}
|
||||
}
|
||||
|
||||
// describeExchangeConfigIssue mirrors the create-time exchange validation but
|
||||
// returns stable uppercase codes for the checklist UI.
|
||||
func describeExchangeConfigIssue(exchange *store.Exchange) (string, string) {
|
||||
if exchange == nil {
|
||||
return "The selected exchange account was not found. Connect an exchange first.", "EXCHANGE_NOT_FOUND"
|
||||
}
|
||||
if !exchange.Enabled {
|
||||
return fmt.Sprintf("Exchange account \"%s\" is disabled. Enable it first.", exchangeDisplayName(exchange)), "EXCHANGE_DISABLED"
|
||||
}
|
||||
if missing := missingExchangeFields(exchange); len(missing) > 0 {
|
||||
return fmt.Sprintf(
|
||||
"Exchange account \"%s\" is missing %s. Complete the connection first.",
|
||||
exchangeDisplayName(exchange), strings.Join(missing, ", "),
|
||||
), "EXCHANGE_MISSING_FIELDS"
|
||||
}
|
||||
if exchange.ExchangeType == "hyperliquid" && !exchange.HyperliquidBuilderApproved {
|
||||
return "Hyperliquid builder authorization is not complete. Reconnect the wallet and finish the authorization.", "HYPERLIQUID_BUILDER_NOT_APPROVED"
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
293
api/launch_preflight_test.go
Normal file
293
api/launch_preflight_test.go
Normal file
@@ -0,0 +1,293 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"nofx/crypto"
|
||||
"nofx/store"
|
||||
)
|
||||
|
||||
// Well-known throwaway development key (hardhat account #1) — never funded.
|
||||
const testClaw402Key = "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
|
||||
|
||||
func withAIWalletBalance(t *testing.T, balance float64, err error) {
|
||||
t.Helper()
|
||||
original := queryAIWalletBalance
|
||||
queryAIWalletBalance = func(string) (float64, error) {
|
||||
return balance, err
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
queryAIWalletBalance = original
|
||||
})
|
||||
}
|
||||
|
||||
func findCheck(t *testing.T, checks []LaunchCheck, id string) LaunchCheck {
|
||||
t.Helper()
|
||||
for _, check := range checks {
|
||||
if check.ID == id {
|
||||
return check
|
||||
}
|
||||
}
|
||||
t.Fatalf("check %q not found in %+v", id, checks)
|
||||
return LaunchCheck{}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIModel(t *testing.T) {
|
||||
if got := checkLaunchAIModel(nil); got.Status != launchCheckStatusFailed || got.Code != "MODEL_NOT_FOUND" {
|
||||
t.Fatalf("nil model: expected failed/MODEL_NOT_FOUND, got %+v", got)
|
||||
}
|
||||
|
||||
disabled := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: false}
|
||||
if got := checkLaunchAIModel(disabled); got.Code != "MODEL_DISABLED" {
|
||||
t.Fatalf("disabled model: expected MODEL_DISABLED, got %+v", got)
|
||||
}
|
||||
|
||||
noKey := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true}
|
||||
if got := checkLaunchAIModel(noKey); got.Code != "MODEL_MISSING_CREDENTIALS" {
|
||||
t.Fatalf("missing key: expected MODEL_MISSING_CREDENTIALS, got %+v", got)
|
||||
}
|
||||
|
||||
ready := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
if got := checkLaunchAIModel(ready); got.Status != launchCheckStatusOK {
|
||||
t.Fatalf("ready model: expected ok, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIWalletSkipsNonClaw402(t *testing.T) {
|
||||
model := &store.AIModel{Name: "DeepSeek", Provider: "deepseek", Enabled: true, APIKey: crypto.EncryptedString("sk-test")}
|
||||
checks := checkLaunchAIWallet(model)
|
||||
if got := findCheck(t, checks, launchCheckAIWallet); got.Status != launchCheckStatusSkipped {
|
||||
t.Fatalf("non-claw402 wallet check should be skipped, got %+v", got)
|
||||
}
|
||||
if got := findCheck(t, checks, launchCheckAIWalletFunds); got.Status != launchCheckStatusSkipped {
|
||||
t.Fatalf("non-claw402 funds check should be skipped, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIWalletInvalidKey(t *testing.T) {
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString("not-a-key")}
|
||||
checks := checkLaunchAIWallet(model)
|
||||
got := findCheck(t, checks, launchCheckAIWallet)
|
||||
if got.Status != launchCheckStatusFailed || got.Code != "AI_WALLET_INVALID_KEY" {
|
||||
t.Fatalf("invalid key: expected failed/AI_WALLET_INVALID_KEY, got %+v", got)
|
||||
}
|
||||
if funds := findCheck(t, checks, launchCheckAIWalletFunds); funds.Status != launchCheckStatusSkipped {
|
||||
t.Fatalf("funds check should be skipped when the key is invalid, got %+v", funds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIWalletInsufficientFunds(t *testing.T) {
|
||||
withAIWalletBalance(t, 0.25, nil)
|
||||
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
checks := checkLaunchAIWallet(model)
|
||||
|
||||
wallet := findCheck(t, checks, launchCheckAIWallet)
|
||||
if wallet.Status != launchCheckStatusOK || wallet.Address == "" {
|
||||
t.Fatalf("wallet check should pass with derived address, got %+v", wallet)
|
||||
}
|
||||
|
||||
funds := findCheck(t, checks, launchCheckAIWalletFunds)
|
||||
if funds.Status != launchCheckStatusFailed || funds.Code != "AI_WALLET_INSUFFICIENT_FUNDS" {
|
||||
t.Fatalf("expected failed/AI_WALLET_INSUFFICIENT_FUNDS, got %+v", funds)
|
||||
}
|
||||
if funds.Actual == nil || *funds.Actual != 0.25 {
|
||||
t.Fatalf("expected actual balance 0.25, got %+v", funds.Actual)
|
||||
}
|
||||
if funds.Required != MinAIFeeUSDC {
|
||||
t.Fatalf("expected required %v, got %v", MinAIFeeUSDC, funds.Required)
|
||||
}
|
||||
if funds.Address == "" {
|
||||
t.Fatalf("funds check should carry the deposit address")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIWalletRPCOutageIsWarningNotFailure(t *testing.T) {
|
||||
withAIWalletBalance(t, 0, errors.New("rpc unreachable"))
|
||||
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
checks := checkLaunchAIWallet(model)
|
||||
|
||||
funds := findCheck(t, checks, launchCheckAIWalletFunds)
|
||||
if funds.Status != launchCheckStatusWarning || funds.Code != "AI_WALLET_BALANCE_UNKNOWN" {
|
||||
t.Fatalf("RPC outage must degrade to warning, got %+v", funds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchAIWalletFunded(t *testing.T) {
|
||||
withAIWalletBalance(t, 25.5, nil)
|
||||
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
checks := checkLaunchAIWallet(model)
|
||||
|
||||
funds := findCheck(t, checks, launchCheckAIWalletFunds)
|
||||
if funds.Status != launchCheckStatusOK {
|
||||
t.Fatalf("funded wallet should pass, got %+v", funds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeExchangeConfigIssue(t *testing.T) {
|
||||
if _, code := describeExchangeConfigIssue(nil); code != "EXCHANGE_NOT_FOUND" {
|
||||
t.Fatalf("nil exchange: expected EXCHANGE_NOT_FOUND, got %s", code)
|
||||
}
|
||||
|
||||
disabled := &store.Exchange{ID: "ex", ExchangeType: "hyperliquid", Enabled: false}
|
||||
if _, code := describeExchangeConfigIssue(disabled); code != "EXCHANGE_DISABLED" {
|
||||
t.Fatalf("disabled: expected EXCHANGE_DISABLED, got %s", code)
|
||||
}
|
||||
|
||||
missing := &store.Exchange{ID: "ex", ExchangeType: "hyperliquid", Enabled: true}
|
||||
if _, code := describeExchangeConfigIssue(missing); code != "EXCHANGE_MISSING_FIELDS" {
|
||||
t.Fatalf("missing fields: expected EXCHANGE_MISSING_FIELDS, got %s", code)
|
||||
}
|
||||
|
||||
unapproved := &store.Exchange{
|
||||
ID: "ex",
|
||||
ExchangeType: "hyperliquid",
|
||||
Enabled: true,
|
||||
APIKey: crypto.EncryptedString(testClaw402Key),
|
||||
HyperliquidWalletAddr: "0x1111111111111111111111111111111111111111",
|
||||
}
|
||||
if _, code := describeExchangeConfigIssue(unapproved); code != "HYPERLIQUID_BUILDER_NOT_APPROVED" {
|
||||
t.Fatalf("builder unapproved: expected HYPERLIQUID_BUILDER_NOT_APPROVED, got %s", code)
|
||||
}
|
||||
|
||||
unapproved.HyperliquidBuilderApproved = true
|
||||
if _, code := describeExchangeConfigIssue(unapproved); code != "" {
|
||||
t.Fatalf("ready exchange: expected no issue, got %s", code)
|
||||
}
|
||||
}
|
||||
|
||||
func readyHyperliquidExchange() *store.Exchange {
|
||||
return &store.Exchange{
|
||||
ID: "ex-hl",
|
||||
ExchangeType: "hyperliquid",
|
||||
Enabled: true,
|
||||
APIKey: crypto.EncryptedString(testClaw402Key),
|
||||
HyperliquidWalletAddr: "0x1111111111111111111111111111111111111111",
|
||||
HyperliquidBuilderApproved: true,
|
||||
}
|
||||
}
|
||||
|
||||
func preflightTestServer(t *testing.T, userID string, states map[string]ExchangeAccountState) *Server {
|
||||
t.Helper()
|
||||
server := &Server{exchangeAccountStateCache: NewExchangeAccountStateCache()}
|
||||
if states != nil {
|
||||
server.exchangeAccountStateCache.Set(userID, states)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
func TestCheckLaunchExchangeInsufficientFundsBlocks(t *testing.T) {
|
||||
exchange := readyHyperliquidExchange()
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 5.5, TotalEquity: 5.5},
|
||||
})
|
||||
|
||||
checks := server.checkLaunchExchange("user-1", exchange)
|
||||
funds := findCheck(t, checks, launchCheckExchangeFunds)
|
||||
if funds.Status != launchCheckStatusFailed || funds.Code != "EXCHANGE_INSUFFICIENT_FUNDS" {
|
||||
t.Fatalf("expected failed/EXCHANGE_INSUFFICIENT_FUNDS, got %+v", funds)
|
||||
}
|
||||
if funds.Actual == nil || *funds.Actual != 5.5 {
|
||||
t.Fatalf("expected actual 5.5, got %+v", funds.Actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchExchangeDeployedMarginPasses(t *testing.T) {
|
||||
// A running bot with capital locked in positions: low available balance
|
||||
// but healthy equity. Restart must not be blocked.
|
||||
exchange := readyHyperliquidExchange()
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 3, TotalEquity: 100},
|
||||
})
|
||||
|
||||
checks := server.checkLaunchExchange("user-1", exchange)
|
||||
funds := findCheck(t, checks, launchCheckExchangeFunds)
|
||||
if funds.Status != launchCheckStatusOK {
|
||||
t.Fatalf("deployed margin with healthy equity should pass, got %+v", funds)
|
||||
}
|
||||
if funds.Actual == nil || *funds.Actual != 100 {
|
||||
t.Fatalf("expected actual 100 (equity), got %+v", funds.Actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchExchangeTestnetLowFundsIsWarning(t *testing.T) {
|
||||
exchange := readyHyperliquidExchange()
|
||||
exchange.Testnet = true
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 0},
|
||||
})
|
||||
|
||||
checks := server.checkLaunchExchange("user-1", exchange)
|
||||
funds := findCheck(t, checks, launchCheckExchangeFunds)
|
||||
if funds.Status != launchCheckStatusWarning {
|
||||
t.Fatalf("testnet low funds should warn, not block, got %+v", funds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckLaunchExchangeInvalidCredentials(t *testing.T) {
|
||||
exchange := readyHyperliquidExchange()
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {
|
||||
ExchangeID: exchange.ID,
|
||||
Status: exchangeAccountStatusInvalidCredentials,
|
||||
ErrorCode: "INVALID_CREDENTIALS",
|
||||
ErrorMessage: "Exchange credentials are invalid",
|
||||
},
|
||||
})
|
||||
|
||||
checks := server.checkLaunchExchange("user-1", exchange)
|
||||
account := findCheck(t, checks, launchCheckExchangeAccount)
|
||||
if account.Status != launchCheckStatusFailed || account.Code != "INVALID_CREDENTIALS" {
|
||||
t.Fatalf("expected failed/INVALID_CREDENTIALS, got %+v", account)
|
||||
}
|
||||
if funds := findCheck(t, checks, launchCheckExchangeFunds); funds.Status != launchCheckStatusSkipped {
|
||||
t.Fatalf("funds check should be skipped when account probe fails, got %+v", funds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLaunchPreflightAggregatesReadiness(t *testing.T) {
|
||||
withAIWalletBalance(t, 10, nil)
|
||||
|
||||
exchange := readyHyperliquidExchange()
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 100, TotalEquity: 100},
|
||||
})
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
strategy := &store.Strategy{ID: "strat-1", Name: "Autopilot"}
|
||||
|
||||
result := server.runLaunchPreflight("user-1", model, exchange, strategy, true)
|
||||
if !result.Ready {
|
||||
t.Fatalf("expected ready, got %+v", result)
|
||||
}
|
||||
if result.MinAIFeeUSDC != MinAIFeeUSDC || result.MinTradingUSDC != MinTradingUSDC {
|
||||
t.Fatalf("minimums must be exposed in the response, got %+v", result)
|
||||
}
|
||||
|
||||
// Break one prerequisite → not ready, and Summary explains it.
|
||||
withAIWalletBalance(t, 0, nil)
|
||||
result = server.runLaunchPreflight("user-1", model, exchange, strategy, true)
|
||||
if result.Ready {
|
||||
t.Fatalf("expected not ready with empty AI wallet, got %+v", result)
|
||||
}
|
||||
if result.Summary() == "" {
|
||||
t.Fatalf("summary should describe the failing check")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLaunchPreflightWarningsDoNotBlock(t *testing.T) {
|
||||
withAIWalletBalance(t, 0, errors.New("rpc down"))
|
||||
|
||||
exchange := readyHyperliquidExchange()
|
||||
server := preflightTestServer(t, "user-1", map[string]ExchangeAccountState{
|
||||
exchange.ID: {ExchangeID: exchange.ID, Status: exchangeAccountStatusOK, AvailableBalance: 100},
|
||||
})
|
||||
model := &store.AIModel{Name: "Claw402", Provider: "claw402", Enabled: true, APIKey: crypto.EncryptedString(testClaw402Key)}
|
||||
|
||||
result := server.runLaunchPreflight("user-1", model, exchange, nil, false)
|
||||
if !result.Ready {
|
||||
t.Fatalf("warnings must not block launch, got %+v", result)
|
||||
}
|
||||
}
|
||||
@@ -237,8 +237,17 @@ Only include fields you want to change.`,
|
||||
`:id = trader_id from GET /api/my-traders. Stops and permanently removes the trader and all its data.`,
|
||||
s.handleDeleteTrader)
|
||||
s.routeWithSchema(protected, "POST", "/traders/:id/start", "Start trader — begins live trading",
|
||||
`:id = trader_id from GET /api/my-traders. No request body needed. The trader must have a valid exchange and AI model configured.`,
|
||||
`:id = trader_id from GET /api/my-traders. No request body needed. The trader must have a valid exchange and AI model configured.
|
||||
Runs launch preflight first and returns 400 with {"error_key":"trader.start.preflight_failed","preflight":{...}} when checks fail. Append ?force=true to skip balance gates.`,
|
||||
s.handleStartTrader)
|
||||
s.routeWithSchema(protected, "GET", "/traders/:id/preflight", "Run launch readiness checks for a trader",
|
||||
`:id = trader_id from GET /api/my-traders.
|
||||
Returns: {"ready":<bool>,"checks":[{"id":"ai_model|ai_wallet|ai_wallet_funds|strategy|exchange_config|exchange_account|exchange_funds","status":"ok|failed|warning|skipped","code":"<string>","message":"<string>","required":<number>,"actual":<number>,"asset":"<string>","address":"<string>"}],"min_ai_fee_usdc":<number>,"min_trading_usdc":<number>}`,
|
||||
s.handleTraderPreflight)
|
||||
s.routeWithSchema(protected, "POST", "/launch/preflight", "Run launch readiness checks before creating a trader",
|
||||
`Body: {"ai_model_id":"<EXACT id from GET /api/models>","exchange_id":"<EXACT id from GET /api/exchanges>","strategy_id":"<optional, EXACT id from GET /api/strategies>"}
|
||||
Returns the same shape as GET /api/traders/:id/preflight. Use this before POST /api/traders to surface every missing prerequisite at once.`,
|
||||
s.handleLaunchPreflight)
|
||||
s.routeWithSchema(protected, "POST", "/traders/:id/stop", "Stop trader — halts live trading",
|
||||
`:id = trader_id from GET /api/my-traders. No request body needed. Gracefully stops the trading loop.`,
|
||||
s.handleStopTrader)
|
||||
|
||||
@@ -177,8 +177,12 @@ type AutoTrader struct {
|
||||
gridState *GridState // Grid trading state (only used when StrategyType == "grid_trading")
|
||||
claw402WalletAddr string // Claw402 wallet address (derived from private key at start)
|
||||
consecutiveAIFailures int // Consecutive AI call failures
|
||||
runtimeHealthMu sync.RWMutex // Guards safe mode + AI wallet health (loop writes, API reads)
|
||||
safeMode bool // Safe mode: no new positions, protect existing ones
|
||||
safeModeReason string // Why safe mode was activated
|
||||
aiWalletStatus string // "ok"|"low"|"empty"|"unknown" — see runtime_health.go
|
||||
aiWalletBalanceUSDC float64 // Last observed Base USDC balance of the claw402 wallet
|
||||
aiWalletCheckedAt time.Time // When the balance was last observed
|
||||
}
|
||||
|
||||
// NewAutoTrader creates an automatic trader
|
||||
@@ -710,7 +714,9 @@ func (at *AutoTrader) runPreLaunchChecks() {
|
||||
balance, err := wallet.QueryUSDCBalance(addr)
|
||||
if err != nil {
|
||||
logger.Warnf("⚠️ [%s] Could not query USDC balance: %v", at.name, err)
|
||||
at.markAIWalletHealthUnknown()
|
||||
} else {
|
||||
at.setAIWalletHealth(balance)
|
||||
// Estimate runway
|
||||
scanMinutes := int(at.config.ScanInterval.Minutes())
|
||||
modelName := at.config.CustomModelName
|
||||
|
||||
@@ -91,6 +91,19 @@ func (at *AutoTrader) GetStatus() map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime health: safe mode + AI fee wallet, so the dashboard can show a
|
||||
// persistent banner instead of the user digging through logs.
|
||||
safeMode, safeModeReason := at.safeModeState()
|
||||
result["safe_mode"] = safeMode
|
||||
if safeModeReason != "" {
|
||||
result["safe_mode_reason"] = safeModeReason
|
||||
}
|
||||
if status, balance, checkedAt := at.aiWalletHealth(); status != "" {
|
||||
result["ai_wallet_status"] = status
|
||||
result["ai_wallet_balance_usdc"] = balance
|
||||
result["ai_wallet_checked_at"] = checkedAt.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
// - never exceeds MaxPositions,
|
||||
// - never doubles a base symbol already held or already in the decision set.
|
||||
func (at *AutoTrader) ensureLongShortCoverage(decisions []kernel.Decision, ctx *kernel.Context, equity float64) []kernel.Decision {
|
||||
if at == nil || ctx == nil || at.safeMode {
|
||||
if at == nil || ctx == nil || at.isSafeMode() {
|
||||
return decisions
|
||||
}
|
||||
if at.config.StrategyConfig == nil || at.config.StrategyConfig.CoinSource.SourceType != "vergex_signal" {
|
||||
|
||||
@@ -2,10 +2,12 @@ package trader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"nofx/kernel"
|
||||
"nofx/logger"
|
||||
"nofx/market"
|
||||
"nofx/mcp/payment"
|
||||
"nofx/provider/hyperliquid"
|
||||
"nofx/store"
|
||||
"nofx/wallet"
|
||||
@@ -141,10 +143,21 @@ func (at *AutoTrader) runCycle() error {
|
||||
record.Success = false
|
||||
record.ErrorMessage = fmt.Sprintf("Failed to get AI decision: %v", err)
|
||||
|
||||
// Payment-layer rejection means the AI fee wallet is definitively out
|
||||
// of funds — surface it as structured health state (GetStatus), not
|
||||
// just a log line.
|
||||
var insufficientFunds *payment.ErrInsufficientFunds
|
||||
if errors.As(err, &insufficientFunds) {
|
||||
at.markAIWalletEmptyFromPayment(insufficientFunds.Balance)
|
||||
record.ErrorMessage = fmt.Sprintf(
|
||||
"AI fee wallet out of funds: balance $%.2f USDC, next call needs ~$%.2f. Top up the Base USDC wallet.",
|
||||
insufficientFunds.Balance, insufficientFunds.Needed,
|
||||
)
|
||||
}
|
||||
|
||||
// Activate safe mode after 3 consecutive failures
|
||||
if at.consecutiveAIFailures >= 3 && !at.safeMode {
|
||||
at.safeMode = true
|
||||
at.safeModeReason = fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err)
|
||||
if at.consecutiveAIFailures >= 3 && !at.isSafeMode() {
|
||||
at.setSafeMode(true, fmt.Sprintf("AI failed %d consecutive times: %v", at.consecutiveAIFailures, err))
|
||||
at.logErrorf("🛡️ SAFE MODE ACTIVATED — AI failed %d times in a row. No new positions will be opened. Existing positions are protected with current stop-loss settings.", at.consecutiveAIFailures)
|
||||
at.logErrorf("🛡️ Reason: %v", err)
|
||||
at.logErrorf("🛡️ Action: Will keep trying AI each cycle. Safe mode auto-deactivates when AI recovers.")
|
||||
@@ -172,7 +185,7 @@ func (at *AutoTrader) runCycle() error {
|
||||
}
|
||||
|
||||
// In safe mode, don't return error — keep the loop running to retry next cycle
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
at.logWarnf("🛡️ Safe mode: skipping this cycle, will retry in %v", at.config.ScanInterval)
|
||||
return nil
|
||||
}
|
||||
@@ -185,10 +198,9 @@ func (at *AutoTrader) runCycle() error {
|
||||
at.logInfof("✅ AI recovered after %d consecutive failures", at.consecutiveAIFailures)
|
||||
}
|
||||
at.consecutiveAIFailures = 0
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
at.logInfof("🛡️ SAFE MODE DEACTIVATED — AI is working again. Resuming normal trading.")
|
||||
at.safeMode = false
|
||||
at.safeModeReason = ""
|
||||
at.setSafeMode(false, "")
|
||||
}
|
||||
|
||||
// // 5. Print system prompt
|
||||
@@ -242,7 +254,7 @@ func (at *AutoTrader) runCycle() error {
|
||||
}
|
||||
|
||||
// Safe mode: filter out open positions, only allow close/hold
|
||||
if at.safeMode {
|
||||
if at.isSafeMode() {
|
||||
filtered := make([]kernel.Decision, 0)
|
||||
for _, d := range sortedDecisions {
|
||||
if d.Action == "open_long" || d.Action == "open_short" {
|
||||
@@ -770,10 +782,12 @@ func (at *AutoTrader) checkClaw402Balance() {
|
||||
balance, err := wallet.QueryUSDCBalance(at.claw402WalletAddr)
|
||||
if err != nil {
|
||||
at.logWarnf("⚠️ Failed to query USDC balance: %v", err)
|
||||
at.markAIWalletHealthUnknown()
|
||||
return
|
||||
}
|
||||
|
||||
if balance < 1.0 {
|
||||
at.setAIWalletHealth(balance)
|
||||
if balance < aiWalletLowThresholdUSDC {
|
||||
at.logWarnf("⚠️ Low USDC balance: $%.2f — AI may stop soon!", balance)
|
||||
}
|
||||
if balance <= 0 {
|
||||
|
||||
82
trader/runtime_health.go
Normal file
82
trader/runtime_health.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package trader
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Runtime health state written by the run-loop goroutine and read by the API
|
||||
// layer (GetStatus). Everything here goes through runtimeHealthMu so the
|
||||
// dashboard can poll without racing the trading loop.
|
||||
|
||||
// AI fee wallet health statuses exposed via GetStatus as "ai_wallet_status".
|
||||
const (
|
||||
AIWalletStatusOK = "ok"
|
||||
AIWalletStatusLow = "low"
|
||||
AIWalletStatusEmpty = "empty"
|
||||
AIWalletStatusUnknown = "unknown"
|
||||
)
|
||||
|
||||
// aiWalletLowThresholdUSDC mirrors api.MinAIFeeUSDC: below this the wallet
|
||||
// cannot reliably pay for the next AI/data calls.
|
||||
const aiWalletLowThresholdUSDC = 1.0
|
||||
|
||||
func (at *AutoTrader) setSafeMode(active bool, reason string) {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.safeMode = active
|
||||
at.safeModeReason = reason
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
func (at *AutoTrader) isSafeMode() bool {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.safeMode
|
||||
}
|
||||
|
||||
func (at *AutoTrader) safeModeState() (bool, string) {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.safeMode, at.safeModeReason
|
||||
}
|
||||
|
||||
// setAIWalletHealth records a fresh balance observation for the claw402 wallet.
|
||||
func (at *AutoTrader) setAIWalletHealth(balance float64) {
|
||||
status := AIWalletStatusOK
|
||||
switch {
|
||||
case balance <= 0:
|
||||
status = AIWalletStatusEmpty
|
||||
case balance < aiWalletLowThresholdUSDC:
|
||||
status = AIWalletStatusLow
|
||||
}
|
||||
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = status
|
||||
at.aiWalletBalanceUSDC = balance
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
// markAIWalletHealthUnknown keeps the last observed balance but flags that the
|
||||
// current reading could not be refreshed (e.g. Base RPC unreachable).
|
||||
func (at *AutoTrader) markAIWalletHealthUnknown() {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = AIWalletStatusUnknown
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
// markAIWalletEmptyFromPayment records a payment-layer rejection: the wallet
|
||||
// definitively could not cover an AI call.
|
||||
func (at *AutoTrader) markAIWalletEmptyFromPayment(balance float64) {
|
||||
at.runtimeHealthMu.Lock()
|
||||
at.aiWalletStatus = AIWalletStatusEmpty
|
||||
at.aiWalletBalanceUSDC = balance
|
||||
at.aiWalletCheckedAt = time.Now().UTC()
|
||||
at.runtimeHealthMu.Unlock()
|
||||
}
|
||||
|
||||
func (at *AutoTrader) aiWalletHealth() (status string, balance float64, checkedAt time.Time) {
|
||||
at.runtimeHealthMu.RLock()
|
||||
defer at.runtimeHealthMu.RUnlock()
|
||||
return at.aiWalletStatus, at.aiWalletBalanceUSDC, at.aiWalletCheckedAt
|
||||
}
|
||||
Reference in New Issue
Block a user