mirror of
https://github.com/NoFxAiOS/nofx.git
synced 2026-07-15 16:56:56 +08:00
feat: migrate timestamps to int64 and security improvements
- Convert all time.Time fields to int64 Unix milliseconds (UTC) - Add PostgreSQL migration to convert timestamp columns to bigint - Reduce Binance sync window from 7 days to 24 hours - Fix dashboard trader name visibility (add nofx-text-main color) - Add position value column to history table - Remove hardcoded API keys from test files
This commit is contained in:
tinkle-community
140
store/order.go
140
store/order.go
@@ -9,36 +9,37 @@ import (
|
||||
)
|
||||
|
||||
// TraderOrder order record
|
||||
// All time fields use int64 millisecond timestamps (UTC) to avoid timezone issues
|
||||
type TraderOrder struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement" json:"id"`
|
||||
TraderID string `gorm:"column:trader_id;not null;index:idx_orders_trader_id" json:"trader_id"`
|
||||
ExchangeID string `gorm:"column:exchange_id;not null;default:''" json:"exchange_id"`
|
||||
ExchangeType string `gorm:"column:exchange_type;not null;default:''" json:"exchange_type"`
|
||||
ExchangeOrderID string `gorm:"column:exchange_order_id;not null;uniqueIndex:idx_orders_exchange_unique,priority:2" json:"exchange_order_id"`
|
||||
ClientOrderID string `gorm:"column:client_order_id;default:''" json:"client_order_id"`
|
||||
Symbol string `gorm:"column:symbol;not null;index:idx_orders_symbol" json:"symbol"`
|
||||
Side string `gorm:"column:side;not null" json:"side"`
|
||||
PositionSide string `gorm:"column:position_side;default:''" json:"position_side"`
|
||||
Type string `gorm:"column:type;not null" json:"type"`
|
||||
TimeInForce string `gorm:"column:time_in_force;default:GTC" json:"time_in_force"`
|
||||
Quantity float64 `gorm:"column:quantity;not null" json:"quantity"`
|
||||
Price float64 `gorm:"column:price;default:0" json:"price"`
|
||||
StopPrice float64 `gorm:"column:stop_price;default:0" json:"stop_price"`
|
||||
Status string `gorm:"column:status;not null;default:NEW;index:idx_orders_status" json:"status"`
|
||||
FilledQuantity float64 `gorm:"column:filled_quantity;default:0" json:"filled_quantity"`
|
||||
AvgFillPrice float64 `gorm:"column:avg_fill_price;default:0" json:"avg_fill_price"`
|
||||
Commission float64 `gorm:"column:commission;default:0" json:"commission"`
|
||||
CommissionAsset string `gorm:"column:commission_asset;default:USDT" json:"commission_asset"`
|
||||
Leverage int `gorm:"column:leverage;default:1" json:"leverage"`
|
||||
ReduceOnly bool `gorm:"column:reduce_only;default:false" json:"reduce_only"`
|
||||
ClosePosition bool `gorm:"column:close_position;default:false" json:"close_position"`
|
||||
WorkingType string `gorm:"column:working_type;default:CONTRACT_PRICE" json:"working_type"`
|
||||
PriceProtect bool `gorm:"column:price_protect;default:false" json:"price_protect"`
|
||||
OrderAction string `gorm:"column:order_action;default:''" json:"order_action"`
|
||||
RelatedPositionID int64 `gorm:"column:related_position_id;default:0" json:"related_position_id"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`
|
||||
FilledAt time.Time `gorm:"column:filled_at" json:"filled_at"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement" json:"id"`
|
||||
TraderID string `gorm:"column:trader_id;not null;index:idx_orders_trader_id" json:"trader_id"`
|
||||
ExchangeID string `gorm:"column:exchange_id;not null;default:''" json:"exchange_id"`
|
||||
ExchangeType string `gorm:"column:exchange_type;not null;default:''" json:"exchange_type"`
|
||||
ExchangeOrderID string `gorm:"column:exchange_order_id;not null;uniqueIndex:idx_orders_exchange_unique,priority:2" json:"exchange_order_id"`
|
||||
ClientOrderID string `gorm:"column:client_order_id;default:''" json:"client_order_id"`
|
||||
Symbol string `gorm:"column:symbol;not null;index:idx_orders_symbol" json:"symbol"`
|
||||
Side string `gorm:"column:side;not null" json:"side"`
|
||||
PositionSide string `gorm:"column:position_side;default:''" json:"position_side"`
|
||||
Type string `gorm:"column:type;not null" json:"type"`
|
||||
TimeInForce string `gorm:"column:time_in_force;default:GTC" json:"time_in_force"`
|
||||
Quantity float64 `gorm:"column:quantity;not null" json:"quantity"`
|
||||
Price float64 `gorm:"column:price;default:0" json:"price"`
|
||||
StopPrice float64 `gorm:"column:stop_price;default:0" json:"stop_price"`
|
||||
Status string `gorm:"column:status;not null;default:NEW;index:idx_orders_status" json:"status"`
|
||||
FilledQuantity float64 `gorm:"column:filled_quantity;default:0" json:"filled_quantity"`
|
||||
AvgFillPrice float64 `gorm:"column:avg_fill_price;default:0" json:"avg_fill_price"`
|
||||
Commission float64 `gorm:"column:commission;default:0" json:"commission"`
|
||||
CommissionAsset string `gorm:"column:commission_asset;default:USDT" json:"commission_asset"`
|
||||
Leverage int `gorm:"column:leverage;default:1" json:"leverage"`
|
||||
ReduceOnly bool `gorm:"column:reduce_only;default:false" json:"reduce_only"`
|
||||
ClosePosition bool `gorm:"column:close_position;default:false" json:"close_position"`
|
||||
WorkingType string `gorm:"column:working_type;default:CONTRACT_PRICE" json:"working_type"`
|
||||
PriceProtect bool `gorm:"column:price_protect;default:false" json:"price_protect"`
|
||||
OrderAction string `gorm:"column:order_action;default:''" json:"order_action"`
|
||||
RelatedPositionID int64 `gorm:"column:related_position_id;default:0" json:"related_position_id"`
|
||||
CreatedAt int64 `gorm:"column:created_at" json:"created_at"` // Unix milliseconds UTC
|
||||
UpdatedAt int64 `gorm:"column:updated_at" json:"updated_at"` // Unix milliseconds UTC
|
||||
FilledAt int64 `gorm:"column:filled_at" json:"filled_at"` // Unix milliseconds UTC
|
||||
}
|
||||
|
||||
// TableName returns the table name for TraderOrder
|
||||
@@ -47,24 +48,25 @@ func (TraderOrder) TableName() string {
|
||||
}
|
||||
|
||||
// TraderFill trade record
|
||||
// All time fields use int64 millisecond timestamps (UTC) to avoid timezone issues
|
||||
type TraderFill struct {
|
||||
ID int64 `gorm:"primaryKey;autoIncrement" json:"id"`
|
||||
TraderID string `gorm:"column:trader_id;not null;index:idx_fills_trader_id" json:"trader_id"`
|
||||
ExchangeID string `gorm:"column:exchange_id;not null;default:''" json:"exchange_id"`
|
||||
ExchangeType string `gorm:"column:exchange_type;not null;default:''" json:"exchange_type"`
|
||||
OrderID int64 `gorm:"column:order_id;not null;index:idx_fills_order_id" json:"order_id"`
|
||||
ExchangeOrderID string `gorm:"column:exchange_order_id;not null" json:"exchange_order_id"`
|
||||
ExchangeTradeID string `gorm:"column:exchange_trade_id;not null;uniqueIndex:idx_fills_exchange_unique,priority:2" json:"exchange_trade_id"`
|
||||
Symbol string `gorm:"column:symbol;not null" json:"symbol"`
|
||||
Side string `gorm:"column:side;not null" json:"side"`
|
||||
Price float64 `gorm:"column:price;not null" json:"price"`
|
||||
Quantity float64 `gorm:"column:quantity;not null" json:"quantity"`
|
||||
QuoteQuantity float64 `gorm:"column:quote_quantity;not null" json:"quote_quantity"`
|
||||
Commission float64 `gorm:"column:commission;not null" json:"commission"`
|
||||
CommissionAsset string `gorm:"column:commission_asset;not null" json:"commission_asset"`
|
||||
RealizedPnL float64 `gorm:"column:realized_pnl;default:0" json:"realized_pnl"`
|
||||
IsMaker bool `gorm:"column:is_maker;default:false" json:"is_maker"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
|
||||
ID int64 `gorm:"primaryKey;autoIncrement" json:"id"`
|
||||
TraderID string `gorm:"column:trader_id;not null;index:idx_fills_trader_id" json:"trader_id"`
|
||||
ExchangeID string `gorm:"column:exchange_id;not null;default:''" json:"exchange_id"`
|
||||
ExchangeType string `gorm:"column:exchange_type;not null;default:''" json:"exchange_type"`
|
||||
OrderID int64 `gorm:"column:order_id;not null;index:idx_fills_order_id" json:"order_id"`
|
||||
ExchangeOrderID string `gorm:"column:exchange_order_id;not null" json:"exchange_order_id"`
|
||||
ExchangeTradeID string `gorm:"column:exchange_trade_id;not null;uniqueIndex:idx_fills_exchange_unique,priority:2" json:"exchange_trade_id"`
|
||||
Symbol string `gorm:"column:symbol;not null" json:"symbol"`
|
||||
Side string `gorm:"column:side;not null" json:"side"`
|
||||
Price float64 `gorm:"column:price;not null" json:"price"`
|
||||
Quantity float64 `gorm:"column:quantity;not null" json:"quantity"`
|
||||
QuoteQuantity float64 `gorm:"column:quote_quantity;not null" json:"quote_quantity"`
|
||||
Commission float64 `gorm:"column:commission;not null" json:"commission"`
|
||||
CommissionAsset string `gorm:"column:commission_asset;not null" json:"commission_asset"`
|
||||
RealizedPnL float64 `gorm:"column:realized_pnl;default:0" json:"realized_pnl"`
|
||||
IsMaker bool `gorm:"column:is_maker;default:false" json:"is_maker"`
|
||||
CreatedAt int64 `gorm:"column:created_at" json:"created_at"` // Unix milliseconds UTC
|
||||
}
|
||||
|
||||
// TableName returns the table name for TraderFill
|
||||
@@ -105,6 +107,23 @@ func (s *OrderStore) InitTables() error {
|
||||
s.db.Exec(fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT false", c.table, c.col))
|
||||
}
|
||||
|
||||
// Migrate timestamp columns to bigint (Unix milliseconds UTC)
|
||||
// Check if column is still timestamp type before migrating
|
||||
timestampColumns := []struct{ table, col string }{
|
||||
{"trader_orders", "created_at"},
|
||||
{"trader_orders", "updated_at"},
|
||||
{"trader_orders", "filled_at"},
|
||||
{"trader_fills", "created_at"},
|
||||
}
|
||||
for _, c := range timestampColumns {
|
||||
var dataType string
|
||||
s.db.Raw(`SELECT data_type FROM information_schema.columns WHERE table_name = ? AND column_name = ?`, c.table, c.col).Scan(&dataType)
|
||||
if dataType == "timestamp with time zone" || dataType == "timestamp without time zone" {
|
||||
// Convert timestamp to Unix milliseconds (bigint)
|
||||
s.db.Exec(fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN %s TYPE BIGINT USING EXTRACT(EPOCH FROM %s) * 1000`, c.table, c.col, c.col))
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure indexes exist
|
||||
s.db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_orders_exchange_unique ON trader_orders(exchange_id, exchange_order_id)`)
|
||||
s.db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_fills_exchange_unique ON trader_fills(exchange_id, exchange_trade_id)`)
|
||||
@@ -153,10 +172,11 @@ func (s *OrderStore) UpdateOrderStatus(id int64, status string, filledQty, avgPr
|
||||
"filled_quantity": filledQty,
|
||||
"avg_fill_price": avgPrice,
|
||||
"commission": commission,
|
||||
"updated_at": time.Now().UTC().UnixMilli(),
|
||||
}
|
||||
|
||||
if status == "FILLED" {
|
||||
updates["filled_at"] = time.Now()
|
||||
updates["filled_at"] = time.Now().UTC().UnixMilli()
|
||||
}
|
||||
|
||||
return s.db.Model(&TraderOrder{}).Where("id = ?", id).Updates(updates).Error
|
||||
@@ -354,3 +374,29 @@ func (s *OrderStore) GetMaxTradeIDsByExchange(exchangeID string) (map[string]int
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetLastFillTimeByExchange returns the most recent fill time (Unix ms) for a given exchange
|
||||
// Used to recover sync state after service restart
|
||||
func (s *OrderStore) GetLastFillTimeByExchange(exchangeID string) (int64, error) {
|
||||
var fill TraderFill
|
||||
err := s.db.Where("exchange_id = ?", exchangeID).
|
||||
Order("created_at DESC").
|
||||
First(&fill).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return fill.CreatedAt, nil
|
||||
}
|
||||
|
||||
// GetRecentFillSymbolsByExchange returns distinct symbols with fills since given time (Unix ms)
|
||||
func (s *OrderStore) GetRecentFillSymbolsByExchange(exchangeID string, sinceMs int64) ([]string, error) {
|
||||
var symbols []string
|
||||
err := s.db.Model(&TraderFill{}).
|
||||
Select("DISTINCT symbol").
|
||||
Where("exchange_id = ? AND created_at >= ?", exchangeID, sinceMs).
|
||||
Pluck("symbol", &symbols).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return symbols, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user