research/stat_pairs_backtest
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new purpose

Browse Source
This commit is contained in:
Oleg Sheynin
2026-07-24 22:40:34 +00:00
parent dc38176529
commit 8ccebf81f5
34 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__SAV__/lib/pt_strategy/live/live_strategy.py
+351
View File
@@ -0,0 +1,351 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.app import App
from cvttpy_tools.base.config import Config
from cvttpy_tools.settings.cvtt_types import IntervalSecT
from cvttpy_tools.base.timeutils import NanosT, SecPerHour, current_nanoseconds, NanoPerSec, format_nanos_utc
from cvttpy_tools.base.logger import Log
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
from cvttpy_trading.trading.mkt_data.md_summary import MdTradesAggregate
from cvttpy_trading.trading.trading_instructions import TradingInstructions
from cvttpy_trading.trading.trading_instructions import TargetPositionSignal
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.pt_model import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import LiveTradingPair
from pairs_trading.apps.pair_trader import PairTrader
from pairs_trading.lib.pt_strategy.pt_market_data import LiveMarketData
class PtLiveStrategy(NamedObject):
config_: Config
instruments_: List[ExchangeInstrument]
interval_sec_: IntervalSecT
history_depth_sec_: IntervalSecT
open_threshold_: float
close_threshold_: float
trading_pair_: LiveTradingPair
model_data_policy_: ModelDataPolicy
pairs_trader_: PairTrader
# for presentation: history of prediction values and trading signals
predictions_df_: pd.DataFrame
trading_signals_df_: pd.DataFrame
allowed_md_lag_sec_: int
def __init__(
self,
config: Config,
pairs_trader: PairTrader,
):
self.config_ = config
self.pairs_trader_ = pairs_trader
self.trading_pair_ = LiveTradingPair(
config=config,
instruments=self.pairs_trader_.instruments_,
)
self.model_data_policy_ = ModelDataPolicy.create(
self.config_,
is_real_time=True,
pair=self.trading_pair_,
)
assert (
self.model_data_policy_ is not None
), f"{self.fname()}: Unable to create ModelDataPolicy"
self.predictions_df_ = pd.DataFrame()
self.trading_signals_df_ = pd.DataFrame()
self.instruments_ = self.pairs_trader_.instruments_
App.instance().add_call(
stage=App.Stage.Config, func=self._on_config(), can_run_now=True
)
async def _on_config(self) -> None:
self.interval_sec_ = self.config_.get_value("interval_sec", 0)
assert self.interval_sec_ > 0, "interval_sec cannot be 0"
self.history_depth_sec_ = (
self.config_.get_value("history_depth_hours", 0) * SecPerHour
)
assert self.history_depth_sec_ > 0, "history_depth_hours cannot be 0"
self.allowed_md_lag_sec_ = self.config_.get_value("allowed_md_lag_sec", 3)
self.open_threshold_ = self.config_.get_value(
"model/disequilibrium/open_trshld", 0.0
)
self.close_threshold_ = self.config_.get_value(
"model/disequilibrium/close_trshld", 0.0
)
assert (
self.open_threshold_ > 0
), "disequilibrium/open_trshld must be greater than 0"
assert (
self.close_threshold_ > 0
), "disequilibrium/close_trshld must be greater than 0"
await self.pairs_trader_.subscribe_md()
def __repr__(self) -> str:
return f"{self.classname()}: trading_pair={self.trading_pair_}, mdp={self.model_data_policy_.__class__.__name__}, "
async def on_mkt_data_hist_snapshot(
self, hist_aggr: List[MdTradesAggregate]
) -> None:
if not self._is_md_actual(hist_aggr=hist_aggr):
return
market_data_df: pd.DataFrame = self._create_md_df(hist_aggr=hist_aggr)
if len(market_data_df) == 0:
Log.warning(f"{self.fname()} Unable to create market data df")
return
self.trading_pair_.market_data_ = market_data_df
Log.info(f"{self.fname()}: Running prediction for pair: {self.trading_pair_}")
prediction = self.trading_pair_.run(
market_data_df, self.model_data_policy_.advance()
)
self.predictions_df_ = pd.concat(
[self.predictions_df_, prediction.to_df()], ignore_index=True
)
trading_instructions: List[TradingInstructions] = (
self._create_trading_instructions(
prediction=prediction, last_row=market_data_df.iloc[-1]
)
)
if trading_instructions is not None:
await self._send_trading_instructions(trading_instructions)
def _is_md_actual(self, hist_aggr: List[MdTradesAggregate]) -> bool:
if len(hist_aggr) == 0:
Log.warning(f"{self.fname()} list of aggregates IS EMPTY")
return False
curr_ns = current_nanoseconds()
# MAYBE check market data length
# at 18:05:01 we should see data for 18:04:00
lag_sec = (curr_ns - hist_aggr[-1].aggr_time_ns_) / NanoPerSec - self.interval_sec()
if lag_sec > self.allowed_md_lag_sec_:
Log.warning(
f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
f" Lagging {int(lag_sec)} > {self.allowed_md_lag_sec_} seconds:"
f"\n{len(hist_aggr)} records"
f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
)
return False
else:
Log.info(
f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
f" Lag {int(lag_sec)} <= {self.allowed_md_lag_sec_} seconds"
f"\n{len(hist_aggr)} records"
f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
)
return True
def _create_md_df(self, hist_aggr: List[MdTradesAggregate]) -> pd.DataFrame:
"""
tstamp time_ns symbol open high low close volume num_trades vwap
0 2025-09-10 11:30:00 1757503800000000000 ADA-USDT 0.8750 0.8750 0.8743 0.8743 50710.500 0 0.874489
1 2025-09-10 11:30:00 1757503800000000000 SOL-USDT 219.9700 219.9800 219.6600 219.7000 2648.582 0 219.787847
2 2025-09-10 11:31:00 1757503860000000000 SOL-USDT 219.7000 219.7300 219.6200 219.6200 1134.886 0 219.663460
3 2025-09-10 11:31:00 1757503860000000000 ADA-USDT 0.8743 0.8745 0.8741 0.8741 10696.400 0 0.874234
4 2025-09-10 11:32:00 1757503920000000000 ADA-USDT 0.8742 0.8742 0.8739 0.8740 18546.900 0 0.874037
"""
rows: List[Dict[str, Any]] = []
for aggr in hist_aggr:
exch_inst = aggr.exch_inst_
rows.append(
{
# convert nanoseconds → tz-aware pandas timestamp
"tstamp": pd.to_datetime(aggr.aggr_time_ns_, unit="ns", utc=True),
"time_ns": aggr.aggr_time_ns_,
"symbol": exch_inst.instrument_id().split("-", 1)[1],
"exchange_id": exch_inst.exchange_id_,
"instrument_id": exch_inst.instrument_id(),
"open": exch_inst.get_price(aggr.open_),
"high": exch_inst.get_price(aggr.high_),
"low": exch_inst.get_price(aggr.low_),
"close": exch_inst.get_price(aggr.close_),
"volume": exch_inst.get_quantity(aggr.volume_),
"num_trades": aggr.num_trades_,
"vwap": exch_inst.get_price(aggr.vwap_),
}
)
source_md_df = pd.DataFrame(
rows,
columns=[
"tstamp",
"time_ns",
"symbol",
"exchange_id",
"instrument_id",
"open",
"high",
"low",
"close",
"volume",
"num_trades",
"vwap",
],
)
# automatic sorting
source_md_df.sort_values(
by=["time_ns", "symbol"],
ascending=True,
inplace=True,
kind="mergesort", # stable sort
)
source_md_df.reset_index(drop=True, inplace=True)
pt_mkt_data = LiveMarketData(config=self.config_, instruments=self.instruments_)
pt_mkt_data.origin_mkt_data_df_ = source_md_df
pt_mkt_data.set_market_data()
return pt_mkt_data.market_data_df_
def interval_sec(self) -> IntervalSecT:
return self.interval_sec_
def history_depth_sec(self) -> IntervalSecT:
return self.history_depth_sec_
async def _send_trading_instructions(
self, trading_instructions: List[TradingInstructions]
) -> None:
for ti in trading_instructions:
Log.info(f"{self.fname()} Sending trading instructions {ti}")
await self.pairs_trader_.ti_sender_.send_trading_instructions(ti)
def _create_trading_instructions(
self, prediction: Prediction, last_row: pd.Series
) -> List[TradingInstructions]:
trd_instructions: List[TradingInstructions] = []
pair = self.trading_pair_
scaled_disequilibrium = prediction.scaled_disequilibrium_
abs_scaled_disequilibrium = abs(scaled_disequilibrium)
if abs_scaled_disequilibrium >= self.open_threshold_:
trd_instructions = self._create_open_trade_instructions(
pair, row=last_row, prediction=prediction
)
elif abs_scaled_disequilibrium <= self.close_threshold_ or pair.to_stop_close_conditions(predicted_row=last_row):
trd_instructions = self._create_close_trade_instructions(
pair, row=last_row # , prediction=prediction
)
return trd_instructions
def _strength(self, scaled_disequilibrium: float) -> float:
# TODO PtLiveStrategy._strength()
return 1.0
def _create_open_trade_instructions(
self, pair: LiveTradingPair, row: pd.Series, prediction: Prediction
) -> List[TradingInstructions]:
diseqlbrm = prediction.disequilibrium_
scaled_disequilibrium = prediction.scaled_disequilibrium_
if diseqlbrm > 0:
side_a = -1
side_b = 1
else:
side_a = 1
side_b = -1
ti_a: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=side_a * self._strength(scaled_disequilibrium),
exchange_id=pair.get_instrument_a().exchange_id_,
base_asset=pair.get_instrument_a().base_asset_id_,
quote_asset=pair.get_instrument_a().quote_asset_id_,
user_data={}
),
)
if not ti_a:
return []
ti_b: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=side_b * self._strength(scaled_disequilibrium),
exchange_id=pair.get_instrument_b().exchange_id_,
base_asset=pair.get_instrument_b().base_asset_id_,
quote_asset=pair.get_instrument_b().quote_asset_id_,
user_data={}
),
)
if not ti_b:
return []
return [ti_a, ti_b]
def _create_close_trade_instructions(
self, pair: LiveTradingPair, row: pd.Series
) -> List[TradingInstructions]:
ti_a: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=0,
exchange_id=pair.get_instrument_a().exchange_id_,
base_asset=pair.get_instrument_a().base_asset_id_,
quote_asset=pair.get_instrument_a().quote_asset_id_,
user_data={}
),
)
if not ti_a:
return []
ti_b: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=0,
exchange_id=pair.get_instrument_b().exchange_id_,
base_asset=pair.get_instrument_b().base_asset_id_,
quote_asset=pair.get_instrument_b().quote_asset_id_,
user_data={}
),
)
if not ti_b:
return []
return [ti_a, ti_b]
__SAV__/lib/pt_strategy/model_data_policy.py
+253
View File
@@ -0,0 +1,253 @@
from __future__ import annotations
import copy
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Dict, Optional, cast
import numpy as np
import pandas as pd
from cvttpy_tools.base.config import Config
@dataclass
class DataWindowParams:
training_size_: int
training_start_index_: int
class ModelDataPolicy(ABC):
config_: Config
current_data_params_: DataWindowParams
count_: int
is_real_time_: bool
def __init__(self, config: Config, *args: Any, **kwargs: Any):
self.config_ = config
self.current_data_params_ = DataWindowParams(
training_size_=config.get_value("model/training_size", 120),
training_start_index_=0,
)
self.count_ = 0
self.is_real_time_ = kwargs.get("is_real_time", False)
@abstractmethod
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
self.count_ += 1
if not self.is_real_time_:
print(self.count_, end="\r")
return self.current_data_params_
@staticmethod
def create(config: Config, *args: Any, **kwargs: Any) -> ModelDataPolicy:
import importlib
model_data_policy_class_name = config.get_value("model/model_data_policy_class", None)
assert model_data_policy_class_name is not None
module_name, class_name = model_data_policy_class_name.rsplit(".", 1)
module = importlib.import_module(module_name)
model_training_data_policy_object = getattr(module, class_name)(
config=config, *args, **kwargs
)
return cast(ModelDataPolicy, model_training_data_policy_object)
class RollingWindowDataPolicy(ModelDataPolicy):
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
self.count_ = 1
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
super().advance(mkt_data_df)
if self.is_real_time_:
self.current_data_params_.training_start_index_ = 0
if mkt_data_df and len(mkt_data_df) > self.curren_data_params_.training_size_:
self.current_data_params_.training_start_index_ = -self.curren_data_params_.training_size_
else:
self.current_data_params_.training_start_index_ += 1
return self.current_data_params_
class OptimizedWndDataPolicy(ModelDataPolicy, ABC):
mkt_data_df_: pd.DataFrame
pair_: TradingPair # type: ignore
min_training_size_: int
max_training_size_: int
end_index_: int
prices_a_: np.ndarray
prices_b_: np.ndarray
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
assert (
kwargs.get("pair") is not None
), "pair must be provided"
assert (config.key_exists("model/max_training_size") and config.key_exists("model/min_training_size")
), "min_training_size and max_training_size must be provided"
self.min_training_size_ = cast(int, config.get_value("model/min_training_size"))
self.max_training_size_ = cast(int, config.get_value("model/max_training_size"))
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
self.pair_ = cast(TradingPair, kwargs.get("pair"))
if "mkt_data" in kwargs:
self.mkt_data_df_ = cast(pd.DataFrame, kwargs.get("mkt_data"))
col_a, col_b = self.pair_.colnames()
self.prices_a_ = np.array(self.mkt_data_df_[col_a])
self.prices_b_ = np.array(self.mkt_data_df_[col_b])
assert self.min_training_size_ < self.max_training_size_
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
super().advance(mkt_data_df)
if mkt_data_df is not None:
self.mkt_data_df_ = mkt_data_df
if self.is_real_time_:
self.end_index_ = len(self.mkt_data_df_) - 1
else:
self.end_index_ = self.current_data_params_.training_start_index_ + self.max_training_size_
if self.end_index_ > len(self.mkt_data_df_) - 1:
self.end_index_ = len(self.mkt_data_df_) - 1
self.current_data_params_.training_start_index_ = self.end_index_ - self.max_training_size_
if self.current_data_params_.training_start_index_ < 0:
self.current_data_params_.training_start_index_ = 0
col_a, col_b = self.pair_.colnames()
self.prices_a_ = np.array(self.mkt_data_df_[col_a])
self.prices_b_ = np.array(self.mkt_data_df_[col_b])
self.current_data_params_ = self.optimize_window_size()
return self.current_data_params_
@abstractmethod
def optimize_window_size(self) -> DataWindowParams:
...
class EGOptimizedWndDataPolicy(OptimizedWndDataPolicy):
'''
# Engle-Granger cointegration test
*** VERY SLOW ***
'''
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
# Run Engle-Granger cointegration test
last_pvalue = 1.0
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
from statsmodels.tsa.stattools import coint # type: ignore
start_index = self.end_index_ - trn_size
series_a = self.prices_a_[start_index : self.end_index_]
series_b = self.prices_b_[start_index : self.end_index_]
eg_pvalue = float(coint(series_a, series_b)[1])
if eg_pvalue < last_pvalue:
last_pvalue = eg_pvalue
result.training_size_ = trn_size
result.training_start_index_ = start_index
# print(
# f"*** DEBUG *** end_index={self.end_index_}, best_trn_size={self.current_data_params_.training_size}, {last_pvalue=}"
# )
return result
class ADFOptimizedWndDataPolicy(OptimizedWndDataPolicy):
# Augmented Dickey-Fuller test
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
from statsmodels.regression.linear_model import OLS
from statsmodels.tools.tools import add_constant
from statsmodels.tsa.stattools import adfuller
last_pvalue = 1.0
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
start_index = self.end_index_ - trn_size
y = self.prices_a_[start_index : self.end_index_]
x = self.prices_b_[start_index : self.end_index_]
# Add constant to x for intercept
x_with_const = add_constant(x)
# OLS regression: y = a + b*x + e
model = OLS(y, x_with_const).fit()
residuals = y - model.predict(x_with_const)
# ADF test on residuals
try:
adf_result = adfuller(residuals, maxlag=1, regression="c")
adf_pvalue = float(adf_result[1])
except Exception as e:
# Handle edge cases with exception (e.g., constant series, etc.)
adf_pvalue = 1.0
if adf_pvalue < last_pvalue:
last_pvalue = adf_pvalue
result.training_size_ = trn_size
result.training_start_index_ = start_index
# print(
# f"*** DEBUG *** end_index={self.end_index_},"
# f" best_trn_size={self.current_data_params_.training_size},"
# f" {last_pvalue=}"
# )
return result
class JohansenOptdWndDataPolicy(OptimizedWndDataPolicy):
# Johansen test
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
from statsmodels.tsa.vector_ar.vecm import coint_johansen
import numpy as np
best_stat = -np.inf
best_trn_size = 0
best_start_index = -1
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
start_index = self.end_index_ - trn_size
series_a = self.prices_a_[start_index:self.end_index_]
series_b = self.prices_b_[start_index:self.end_index_]
# Combine into 2D matrix for Johansen test
try:
data = np.column_stack([series_a, series_b])
# Johansen test: det_order=0 (no deterministic trend), k_ar_diff=1 (lag)
res = coint_johansen(data, det_order=0, k_ar_diff=1)
# Trace statistic for cointegration rank 1
trace_stat = res.lr1[0] # test stat for rank=0 vs >=1
critical_value = res.cvt[0, 1] # 5% critical value
if trace_stat > best_stat:
best_stat = trace_stat
best_trn_size = trn_size
best_start_index = start_index
except Exception:
continue
if best_trn_size > 0:
result.training_size_ = best_trn_size
result.training_start_index_ = best_start_index
else:
print("*** WARNING: No valid cointegration window found.")
# print(
# f"*** DEBUG *** end_index={self.end_index_}, best_trn_size={best_trn_size}, trace_stat={best_stat}"
# )
return result
__SAV__/lib/pt_strategy/models.py
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
from typing import Optional
import pandas as pd
import statsmodels.api as sm
from pairs_trading.lib.pt_strategy.pt_model import PairsTradingModel, Prediction
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
class OLSModel(PairsTradingModel):
model_: Optional[sm.regression.linear_model.RegressionResultsWrapper]
pair_predict_result_: Optional[pd.DataFrame]
zscore_df_: Optional[pd.DataFrame]
def predict(self, pair: TradingPair) -> Prediction:
self.training_df_ = pair.market_data_.copy()
zscore_df = self._fit_zscore(pair=pair)
assert zscore_df is not None
# zscore is both disequilibrium and scaled_disequilibrium
self.training_df_["dis-equilibrium"] = zscore_df[0]
self.training_df_["scaled_dis-equilibrium"] = zscore_df[0]
assert zscore_df is not None
return Prediction(
tstamp=pair.market_data_.iloc[-1]["tstamp"],
disequilibrium=self.training_df_["dis-equilibrium"].iloc[-1],
scaled_disequilibrium=self.training_df_["scaled_dis-equilibrium"].iloc[-1],
)
def _fit_zscore(self, pair: TradingPair) -> pd.DataFrame:
assert self.training_df_ is not None
symbol_a_px_series = self.training_df_[pair.colnames()].iloc[:, 0]
symbol_b_px_series = self.training_df_[pair.colnames()].iloc[:, 1]
symbol_a_px_series, symbol_b_px_series = symbol_a_px_series.align(
symbol_b_px_series, axis=0
)
X = sm.add_constant(symbol_b_px_series)
self.model_ = sm.OLS(symbol_a_px_series, X).fit()
assert self.model_ is not None
# alternate way would be to use models residuals (will give identical results)
# alpha, beta = self.model_.params
# spread = symbol_a_px_series - (alpha + beta * symbol_b_px_series)
spread = self.model_.resid
return pd.DataFrame((spread - spread.mean()) / spread.std())
class VECMModel(PairsTradingModel):
def predict(self, pair: TradingPair) -> Prediction:
self.training_df_ = pair.market_data_.copy()
assert self.training_df_ is not None
vecm_fit = self._fit_VECM(pair=pair)
assert vecm_fit is not None
predicted_prices = vecm_fit.predict(steps=1)
# Convert prediction to a DataFrame for readability
predicted_df = pd.DataFrame(
predicted_prices, columns=pd.Index(pair.colnames()), dtype=float
)
disequilibrium = (predicted_df[pair.colnames()] @ vecm_fit.beta)[0][0]
scaled_disequilibrium = (disequilibrium - self.training_mu_) / self.training_std_
return Prediction(
tstamp=pair.market_data_.iloc[-1]["tstamp"],
disequilibrium=disequilibrium,
scaled_disequilibrium=scaled_disequilibrium,
)
def _fit_VECM(self, pair: TradingPair) -> VECMResults: # type: ignore
from statsmodels.tsa.vector_ar.vecm import VECM, VECMResults
vecm_df = self.training_df_[pair.colnames()].reset_index(drop=True)
vecm_model = VECM(vecm_df, coint_rank=1)
vecm_fit = vecm_model.fit()
assert vecm_fit is not None
# Check if the model converged properly
if not hasattr(vecm_fit, "beta") or vecm_fit.beta is None:
print(f"{self}: VECM model failed to converge properly")
diseq_series = self.training_df_[pair.colnames()] @ vecm_fit.beta
# print(diseq_series.shape)
self.training_mu_ = float(diseq_series[0].mean())
self.training_std_ = float(diseq_series[0].std())
self.training_df_["dis-equilibrium"] = (
self.training_df_[pair.colnames()] @ vecm_fit.beta
)
# Normalize the dis-equilibrium
self.training_df_["scaled_dis-equilibrium"] = (
diseq_series - self.training_mu_
) / self.training_std_
return vecm_fit
__SAV__/lib/pt_strategy/prediction.py
+28
View File
@@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Any, Dict
import pandas as pd
class Prediction:
tstamp_: pd.Timestamp
disequilibrium_: float
scaled_disequilibrium_: float
def __init__(self, tstamp: pd.Timestamp, disequilibrium: float, scaled_disequilibrium: float):
self.tstamp_ = tstamp
self.disequilibrium_ = disequilibrium
self.scaled_disequilibrium_ = scaled_disequilibrium
def to_dict(self) -> Dict[str, Any]:
return {
"tstamp": self.tstamp_,
"disequilibrium": self.disequilibrium_,
"signed_scaled_disequilibrium": self.scaled_disequilibrium_,
"scaled_disequilibrium": abs(self.scaled_disequilibrium_),
# "pair": self.pair_,
}
def to_df(self) -> pd.DataFrame:
return pd.DataFrame([self.to_dict()])
__SAV__/lib/pt_strategy/pt_market_data.py
+223
View File
@@ -0,0 +1,223 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.config import Config
from cvttpy_tools.settings.cvtt_types import JsonDictT
# ---
from cvttpy_trading.trading.mkt_data.md_summary import MdTradesAggregate
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.tools.data_loader import load_market_data
class PtMarketData(NamedObject, ABC):
config_: Config
origin_mkt_data_df_: pd.DataFrame
market_data_df_: pd.DataFrame
stat_model_price_: str
instruments_: List[ExchangeInstrument]
symbol_a_: str
symbol_b_: str
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
self.config_ = config
self.origin_mkt_data_df_ = pd.DataFrame()
self.market_data_df_ = pd.DataFrame()
self.stat_model_price_ = self.config_.get_value("model/stat_model_price")
self.instruments_ = instruments
assert len(self.instruments_) > 0, "No instruments found in config"
self.symbol_a_ = self.instruments_[0].instrument_id().split("-", 1)[1]
self.symbol_b_ = self.instruments_[1].instrument_id().split("-", 1)[1]
@abstractmethod
def md_columns(self) -> List[str]: ...
@abstractmethod
def rename_columns(self, symbol_df: pd.DataFrame) -> pd.DataFrame: ...
@abstractmethod
def tranform_df_target_colnames(self) -> List[str]: ...
def set_market_data(self) -> None:
self.market_data_df_ = pd.DataFrame(
self._transform_dataframe(self.origin_mkt_data_df_)[
["tstamp"] + self.tranform_df_target_colnames()
]
)
self.market_data_df_ = self.market_data_df_.dropna().reset_index(drop=True)
self.market_data_df_["tstamp"] = pd.to_datetime(self.market_data_df_["tstamp"])
self.market_data_df_ = self.market_data_df_.sort_values("tstamp")
def colnames(self) -> List[str]:
return [
f"{self.stat_model_price_}_{self.symbol_a_}",
f"{self.stat_model_price_}_{self.symbol_b_}",
]
def _transform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
df_selected: pd.DataFrame = pd.DataFrame(df[self.md_columns()])
result_df = (
pd.DataFrame(df_selected["tstamp"]).drop_duplicates().reset_index(drop=True)
)
# For each unique symbol, add a corresponding stat_model_price column
symbols = df_selected["symbol"].unique()
for symbol in symbols:
# Filter rows for this symbol
df_symbol = df_selected[df_selected["symbol"] == symbol].reset_index(
drop=True
)
# Create column name like "close-COIN"
temp_df: pd.DataFrame = self.rename_columns(df_symbol)
# Join with our result dataframe
result_df = pd.merge(result_df, temp_df, on="tstamp", how="left")
result_df = result_df.reset_index(
drop=True
) # do not dropna() since irrelevant symbol would affect dataset
return result_df.dropna()
class ResearchMarketData(PtMarketData):
current_index_: int
is_execution_price_: bool
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
self.current_index_ = 0
self.is_execution_price_ = self.config_.key_exists("execution_price")
if self.is_execution_price_:
self.execution_price_column_ = self.config_.get_value("execution_price")["column"]
self.execution_price_shift_ = self.config_.get_value("execution_price")["shift"]
else:
self.execution_price_column_ = None
self.execution_price_shift_ = 0
def has_next(self) -> bool:
return self.current_index_ < len(self.market_data_df_)
def get_next(self) -> pd.Series:
result = self.market_data_df_.iloc[self.current_index_]
self.current_index_ += 1
return result
def load(self) -> None:
datafiles: List[str] = self.config_.get_value("datafiles", [])
assert len(datafiles) > 0, "No datafiles found in config"
extra_minutes: int = self.execution_price_shift_
for datafile in datafiles:
md_df = load_market_data(
datafile=datafile,
instruments=self.instruments_,
db_table_name=self.config_.get_value("market_data_loading")[
self.instruments_[0].user_data_.get("instrument_type", "?instrument_type?")
]["db_table_name"],
trading_hours=self.config_.get_value("trading_hours"),
extra_minutes=extra_minutes,
)
self.origin_mkt_data_df_ = pd.concat([self.origin_mkt_data_df_, md_df])
self.origin_mkt_data_df_ = self.origin_mkt_data_df_.sort_values(by="tstamp")
self.origin_mkt_data_df_ = self.origin_mkt_data_df_.dropna().reset_index(
drop=True
)
self.set_market_data()
self._set_execution_price_data()
def _set_execution_price_data(self) -> None:
if not self.is_execution_price_:
return
if not self.config_.key_exists("execution_price"):
self.market_data_df_[f"exec_price_{self.symbol_a_}"] = self.market_data_df_[
f"{self.stat_model_price_}_{self.symbol_a_}"
]
self.market_data_df_[f"exec_price_{self.symbol_b_}"] = self.market_data_df_[
f"{self.stat_model_price_}_{self.symbol_b_}"
]
return
execution_price_column = self.config_.get_value("execution_price")["column"]
execution_price_shift = self.config_.get_value("execution_price")["shift"]
self.market_data_df_[f"exec_price_{self.symbol_a_}"] = self.market_data_df_[
f"{execution_price_column}_{self.symbol_a_}"
].shift(-execution_price_shift)
self.market_data_df_[f"exec_price_{self.symbol_b_}"] = self.market_data_df_[
f"{execution_price_column}_{self.symbol_b_}"
].shift(-execution_price_shift)
self.market_data_df_ = self.market_data_df_.dropna().reset_index(drop=True)
def md_columns(self) -> List[str]:
# @abstractmethod
if self.is_execution_price_:
return ["tstamp", "symbol", self.stat_model_price_, self.execution_price_column_]
else:
return ["tstamp", "symbol", self.stat_model_price_]
def rename_columns(self, selected_symbol_df: pd.DataFrame) -> pd.DataFrame:
# @abstractmethod
symbol = selected_symbol_df.iloc[0]["symbol"]
new_price_column = f"{self.stat_model_price_}_{symbol}"
if self.is_execution_price_:
new_execution_price_column = f"{self.execution_price_column_}_{symbol}"
# Create temporary dataframe with timestamp and price
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
new_execution_price_column: selected_symbol_df[self.execution_price_column_],
}
)
else:
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
}
)
return temp_df
def tranform_df_target_colnames(self):
# @abstractmethod
return self.colnames() + self.orig_exec_prices_colnames()
def orig_exec_prices_colnames(self) -> List[str]:
return [
f"{self.execution_price_column_}_{self.symbol_a_}",
f"{self.execution_price_column_}_{self.symbol_b_}",
] if self.is_execution_price_ else []
class LiveMarketData(PtMarketData):
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
def md_columns(self) -> List[str]:
# @abstractmethod
return ["tstamp", "symbol", self.stat_model_price_]
def rename_columns(self, selected_symbol_df: pd.DataFrame) -> pd.DataFrame:
# @abstractmethod
symbol = selected_symbol_df.iloc[0]["symbol"]
new_price_column = f"{self.stat_model_price_}_{symbol}"
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
}
)
return temp_df
def tranform_df_target_colnames(self):
# @abstractmethod
return self.colnames()
__SAV__/lib/pt_strategy/pt_model.py
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Dict, cast
# ---
from cvttpy_tools.base.config import Config
# ---
from pairs_trading.lib.pt_strategy.prediction import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
class PairsTradingModel(ABC):
@abstractmethod
def predict(self, pair: TradingPair) -> Prediction: # type: ignore[assignment]
...
@staticmethod
def create(config: Config) -> PairsTradingModel:
import importlib
model_class_name = config.get_value("model/model_class", None)
assert model_class_name is not None
module_name, class_name = model_class_name.rsplit(".", 1)
module = importlib.import_module(module_name)
model_object = getattr(module, class_name)()
return cast(PairsTradingModel, model_object)
__SAV__/lib/pt_strategy/research_strategy.py
+305
View File
@@ -0,0 +1,305 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
# ---
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.pt_market_data import ResearchMarketData
from pairs_trading.lib.pt_strategy.pt_model import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import PairState, TradingPair, ResearchTradingPair
class PtResearchStrategy:
config_: Config
trading_pair_: ResearchTradingPair
model_data_policy_: ModelDataPolicy
pt_mkt_data_: ResearchMarketData
trades_: List[pd.DataFrame]
predictions_df_: pd.DataFrame
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument]
):
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
self.config_ = config
self.trades_ = []
self.trading_pair_ = ResearchTradingPair(config=config, instruments=instruments)
self.predictions_df_ = pd.DataFrame()
import copy
# modified config must be passed to PtMarketData
config_copy = copy.deepcopy(config)
config_copy.set_value("instruments", instruments)
self.pt_mkt_data_ = ResearchMarketData(config=config_copy, instruments=instruments)
self.pt_mkt_data_.load()
self.model_data_policy_ = ModelDataPolicy.create(
config_copy, mkt_data=self.pt_mkt_data_.market_data_df_, pair=self.trading_pair_
)
def outstanding_positions(self) -> List[Dict[str, Any]]:
return list(self.trading_pair_.user_data_.get("outstanding_positions", []))
def run(self) -> None:
training_minutes = self.config_.get_value("training_minutes", 120)
market_data_series: pd.Series
market_data_df = pd.DataFrame()
idx = 0
while self.pt_mkt_data_.has_next():
market_data_series = self.pt_mkt_data_.get_next()
new_row = pd.DataFrame([market_data_series])
market_data_df = pd.concat([market_data_df, new_row], ignore_index=True)
if idx >= training_minutes:
break
idx += 1
assert idx >= training_minutes, "Not enough training data"
while self.pt_mkt_data_.has_next():
market_data_series = self.pt_mkt_data_.get_next()
new_row = pd.DataFrame([market_data_series])
market_data_df = pd.concat([market_data_df, new_row], ignore_index=True)
prediction = self.trading_pair_.run(
market_data_df, self.model_data_policy_.advance(mkt_data_df=market_data_df)
)
self.predictions_df_ = pd.concat(
[self.predictions_df_, prediction.to_df()], ignore_index=True
)
assert prediction is not None
trades = self._create_trades(
prediction=prediction, last_row=market_data_df.iloc[-1]
)
if trades is not None:
self.trades_.append(trades)
trades = self._handle_outstanding_positions()
if trades is not None:
self.trades_.append(trades)
def _create_trades(
self, prediction: Prediction, last_row: pd.Series
) -> Optional[pd.DataFrame]:
pair = self.trading_pair_
trades = None
open_threshold = self.config_.get_value("model/disequilibrium/open_trshld")
close_threshold = self.config_.get_value("model/disequilibrium/close_trshld")
scaled_disequilibrium = prediction.scaled_disequilibrium_
abs_scaled_disequilibrium = abs(scaled_disequilibrium)
if pair.user_data_["state"] in [
PairState.INITIAL,
PairState.CLOSE,
PairState.CLOSE_POSITION,
PairState.CLOSE_STOP_LOSS,
PairState.CLOSE_STOP_PROFIT,
]:
if abs_scaled_disequilibrium >= open_threshold:
trades = self._create_open_trades(
pair, row=last_row, prediction=prediction
)
if trades is not None:
trades["status"] = PairState.OPEN.name
print(f"OPEN TRADES:\n{trades}")
pair.user_data_["state"] = PairState.OPEN
pair.on_open_trades(trades)
elif pair.user_data_["state"] == PairState.OPEN:
if abs_scaled_disequilibrium <= close_threshold:
trades = self._create_close_trades(
pair, row=last_row, prediction=prediction
)
if trades is not None:
trades["status"] = PairState.CLOSE.name
print(f"CLOSE TRADES:\n{trades}")
pair.user_data_["state"] = PairState.CLOSE
pair.on_close_trades(trades)
elif pair.to_stop_close_conditions(predicted_row=last_row):
trades = self._create_close_trades(pair, row=last_row)
if trades is not None:
trades["status"] = pair.user_data_["stop_close_state"].name
print(f"STOP CLOSE TRADES:\n{trades}")
pair.user_data_["state"] = pair.user_data_["stop_close_state"]
pair.on_close_trades(trades)
return trades
def _handle_outstanding_positions(self) -> Optional[pd.DataFrame]:
trades = None
pair = self.trading_pair_
# Outstanding positions
if pair.user_data_["state"] == PairState.OPEN:
print(f"{pair}: *** Position is NOT CLOSED. ***")
# outstanding positions
if self.config_.get_value("close_outstanding_positions", False):
close_position_row = pd.Series(pair.market_data_.iloc[-2])
# close_position_row["disequilibrium"] = 0.0
# close_position_row["scaled_disequilibrium"] = 0.0
# close_position_row["signed_scaled_disequilibrium"] = 0.0
trades = self._create_close_trades(
pair=pair, row=close_position_row, prediction=None
)
if trades is not None:
trades["status"] = PairState.CLOSE_POSITION.name
print(f"CLOSE_POSITION TRADES:\n{trades}")
pair.user_data_["state"] = PairState.CLOSE_POSITION
pair.on_close_trades(trades)
else:
pair.add_outstanding_position(
symbol=pair.symbol_a(),
open_side=pair.user_data_["open_side_a"],
open_px=pair.user_data_["open_px_a"],
open_tstamp=pair.user_data_["open_tstamp"],
last_mkt_data_row=pair.market_data_.iloc[-1],
)
pair.add_outstanding_position(
symbol=pair.symbol_b(),
open_side=pair.user_data_["open_side_b"],
open_px=pair.user_data_["open_px_b"],
open_tstamp=pair.user_data_["open_tstamp"],
last_mkt_data_row=pair.market_data_.iloc[-1],
)
return trades
def _trades_df(self) -> pd.DataFrame:
types = {
"time": "datetime64[ns]",
"action": "string",
"symbol": "string",
"side": "string",
"price": "float64",
"disequilibrium": "float64",
"scaled_disequilibrium": "float64",
"signed_scaled_disequilibrium": "float64",
# "pair": "object",
}
columns = list(types.keys())
return pd.DataFrame(columns=columns).astype(types)
def _create_open_trades(
self, pair: ResearchTradingPair, row: pd.Series, prediction: Prediction
) -> Optional[pd.DataFrame]:
colname_a, colname_b = pair.exec_prices_colnames()
tstamp = row["tstamp"]
diseqlbrm = prediction.disequilibrium_
scaled_disequilibrium = prediction.scaled_disequilibrium_
px_a = row[f"{colname_a}"]
px_b = row[f"{colname_b}"]
# creating the trades
df = self._trades_df()
print(f"OPEN_TRADES: {row["tstamp"]} {scaled_disequilibrium=}")
if diseqlbrm > 0:
side_a = "SELL"
side_b = "BUY"
else:
side_a = "BUY"
side_b = "SELL"
# save closing sides
pair.user_data_["open_side_a"] = side_a # used in oustanding positions
pair.user_data_["open_side_b"] = side_b
pair.user_data_["open_px_a"] = px_a
pair.user_data_["open_px_b"] = px_b
pair.user_data_["open_tstamp"] = tstamp
pair.user_data_["close_side_a"] = side_b # used for closing trades
pair.user_data_["close_side_b"] = side_a
# create opening trades
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_a(),
"side": side_a,
"action": "OPEN",
"price": px_a,
"disequilibrium": diseqlbrm,
"signed_scaled_disequilibrium": scaled_disequilibrium,
"scaled_disequilibrium": abs(scaled_disequilibrium),
# "pair": pair,
}
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_b(),
"side": side_b,
"action": "OPEN",
"price": px_b,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": abs(scaled_disequilibrium),
"signed_scaled_disequilibrium": scaled_disequilibrium,
# "pair": pair,
}
return df
def _create_close_trades(
self, pair: ResearchTradingPair, row: pd.Series, prediction: Optional[Prediction] = None
) -> Optional[pd.DataFrame]:
colname_a, colname_b = pair.exec_prices_colnames()
tstamp = row["tstamp"]
if prediction is not None:
diseqlbrm = prediction.disequilibrium_
signed_scaled_disequilibrium = prediction.scaled_disequilibrium_
scaled_disequilibrium = abs(prediction.scaled_disequilibrium_)
else:
diseqlbrm = 0.0
signed_scaled_disequilibrium = 0.0
scaled_disequilibrium = 0.0
px_a = row[f"{colname_a}"]
px_b = row[f"{colname_b}"]
# creating the trades
df = self._trades_df()
# create opening trades
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_a(),
"side": pair.user_data_["close_side_a"],
"action": "CLOSE",
"price": px_a,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": scaled_disequilibrium,
"signed_scaled_disequilibrium": signed_scaled_disequilibrium,
# "pair": pair,
}
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_b(),
"side": pair.user_data_["close_side_b"],
"action": "CLOSE",
"price": px_b,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": scaled_disequilibrium,
"signed_scaled_disequilibrium": signed_scaled_disequilibrium,
# "pair": pair,
}
del pair.user_data_["close_side_a"]
del pair.user_data_["close_side_b"]
del pair.user_data_["open_tstamp"]
del pair.user_data_["open_px_a"]
del pair.user_data_["open_px_b"]
del pair.user_data_["open_side_a"]
del pair.user_data_["open_side_b"]
return df
def day_trades(self) -> pd.DataFrame:
return pd.concat(self.trades_, ignore_index=True)
__SAV__/lib/pt_strategy/results.py
+527
View File
@@ -0,0 +1,527 @@
import os
import sqlite3
from datetime import date, datetime
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
# ---
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
# Recommended replacement adapters and converters for Python 3.12+
# From: https://docs.python.org/3/library/sqlite3.html#sqlite3-adapter-converter-recipes
def adapt_date_iso(val: date) -> str:
"""Adapt datetime.date to ISO 8601 date."""
return val.isoformat()
def adapt_datetime_iso(val: datetime) -> str:
"""Adapt datetime.datetime to timezone-naive ISO 8601 date."""
return val.isoformat()
def convert_date(val: bytes) -> date:
"""Convert ISO 8601 date to datetime.date object."""
return datetime.fromisoformat(val.decode()).date()
def convert_datetime(val: bytes) -> datetime:
"""Convert ISO 8601 datetime to datetime.datetime object."""
return datetime.fromisoformat(val.decode())
# Register the adapters and converters
sqlite3.register_adapter(date, adapt_date_iso)
sqlite3.register_adapter(datetime, adapt_datetime_iso)
sqlite3.register_converter("date", convert_date)
sqlite3.register_converter("datetime", convert_datetime)
def create_result_database(db_path: str) -> None:
"""
Create the SQLite database and required tables if they don't exist.
"""
try:
# Create directory if it doesn't exist
db_dir = os.path.dirname(db_path)
if db_dir and not os.path.exists(db_dir):
os.makedirs(db_dir, exist_ok=True)
print(f"Created directory: {db_dir}")
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Create the pt_bt_results table for completed trades
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS pt_bt_results (
date DATE,
pair TEXT,
symbol TEXT,
open_time DATETIME,
open_side TEXT,
open_price REAL,
open_quantity INTEGER,
open_disequilibrium REAL,
close_time DATETIME,
close_side TEXT,
close_price REAL,
close_quantity INTEGER,
close_disequilibrium REAL,
symbol_return REAL,
pair_return REAL,
close_condition TEXT
)
"""
)
cursor.execute("DELETE FROM pt_bt_results;")
# Create the outstanding_positions table for open positions
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS outstanding_positions (
date DATE,
pair TEXT,
symbol TEXT,
position_quantity REAL,
last_price REAL,
unrealized_return REAL,
open_price REAL,
open_side TEXT
)
"""
)
cursor.execute("DELETE FROM outstanding_positions;")
# Create the config table for storing configuration JSON for reference
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_timestamp DATETIME,
config_file_path TEXT,
config_json TEXT,
datafiles TEXT,
instruments TEXT
)
"""
)
cursor.execute("DELETE FROM config;")
conn.commit()
conn.close()
except Exception as e:
print(f"Error creating result database: {str(e)}")
raise
def store_config_in_database(
db_path: str,
config_file_path: str,
config: Config,
datafiles: List[Tuple[str, str]],
instruments: List[ExchangeInstrument],
) -> None:
"""
Store configuration information in the database for reference.
"""
import json
if db_path.upper() == "NONE":
return
try:
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Convert config to JSON string
config_json = json.dumps(config.data(), indent=2, default=str)
# Convert lists to comma-separated strings for storage
datafiles_str = ", ".join([f"{datafile}" for _, datafile in datafiles])
instruments_str = ", ".join(
[
inst.details_short()
for inst in instruments
]
)
# Insert configuration record
cursor.execute(
"""
INSERT INTO config (
run_timestamp, config_file_path, config_json, datafiles, instruments
) VALUES (?, ?, ?, ?, ?)
""",
(
datetime.now(),
config_file_path,
config_json,
datafiles_str,
instruments_str,
),
)
conn.commit()
conn.close()
print(f"Configuration stored in database")
except Exception as e:
print(f"Error storing configuration in database: {str(e)}")
import traceback
traceback.print_exc()
def convert_timestamp(timestamp: Any) -> Optional[datetime]:
"""Convert pandas Timestamp to Python datetime object for SQLite compatibility."""
if timestamp is None:
return None
if isinstance(timestamp, pd.Timestamp):
return timestamp.to_pydatetime()
elif isinstance(timestamp, datetime):
return timestamp
elif isinstance(timestamp, date):
return datetime.combine(timestamp, datetime.min.time())
elif isinstance(timestamp, str):
return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S")
elif isinstance(timestamp, int):
return datetime.fromtimestamp(timestamp)
else:
raise ValueError(f"Unsupported timestamp type: {type(timestamp)}")
DayT = str
TradeT = Dict[str, Any]
OutstandingPositionT = Dict[str, Any]
class PairResearchResult:
"""
Class to handle pair research results for a single pair across multiple days.
Simplified version of BacktestResult focused on single pair analysis.
"""
trades_: Dict[DayT, pd.DataFrame]
outstanding_positions_: Dict[DayT, List[OutstandingPositionT]]
symbol_roundtrip_trades_: Dict[str, List[Dict[str, Any]]]
config_: Config
def __init__(self, config: Config) -> None:
self.config_ = config
self.trades_ = {}
self.outstanding_positions_ = {}
self.total_realized_pnl = 0.0
self.symbol_roundtrip_trades_ = {}
def add_day_results(self, day: DayT, trades: pd.DataFrame, outstanding_positions: List[Dict[str, Any]]) -> None:
assert isinstance(trades, pd.DataFrame)
self.trades_[day] = trades
self.outstanding_positions_[day] = outstanding_positions
def outstanding_positions(self) -> List[OutstandingPositionT]:
"""Get all outstanding positions across all days as a flat list."""
res: List[Dict[str, Any]] = []
for day in self.outstanding_positions_.keys():
res.extend(self.outstanding_positions_[day])
return res
def calculate_returns(self) -> None:
"""Calculate and store total returns for the single pair across all days."""
self.extract_roundtrip_trades()
self.total_realized_pnl = 0.0
for day, day_trades in self.symbol_roundtrip_trades_.items():
for trade in day_trades:
self.total_realized_pnl += trade['symbol_return']
def extract_roundtrip_trades(self) -> None:
"""
Extract round-trip trades by day, grouping open/close pairs for each symbol.
Returns a dictionary with day as key and list of completed round-trip trades.
"""
def _symbol_return(trade1_side: str, trade1_px: float, trade2_side: str, trade2_px: float) -> float:
if trade1_side == "BUY" and trade2_side == "SELL":
return (trade2_px - trade1_px) / trade1_px * 100
elif trade1_side == "SELL" and trade2_side == "BUY":
return (trade1_px - trade2_px) / trade1_px * 100
else:
return 0
# Process each day separately
for day, day_trades in self.trades_.items():
# Sort trades by timestamp for the day
sorted_trades = day_trades #sorted(day_trades, key=lambda x: x["timestamp"] if x["timestamp"] else pd.Timestamp.min)
day_roundtrips = []
# Process trades in groups of 4 (open A, open B, close A, close B)
for idx in range(0, len(sorted_trades), 4):
if idx + 3 >= len(sorted_trades):
break
trade_a_1 = sorted_trades.iloc[idx] # Open A
trade_b_1 = sorted_trades.iloc[idx + 1] # Open B
trade_a_2 = sorted_trades.iloc[idx + 2] # Close A
trade_b_2 = sorted_trades.iloc[idx + 3] # Close B
# Validate trade sequence
if not (trade_a_1["action"] == "OPEN" and trade_a_2["action"] == "CLOSE"):
continue
if not (trade_b_1["action"] == "OPEN" and trade_b_2["action"] == "CLOSE"):
continue
# Calculate individual symbol returns
symbol_a_return = _symbol_return(
trade_a_1["side"], trade_a_1["price"],
trade_a_2["side"], trade_a_2["price"]
)
symbol_b_return = _symbol_return(
trade_b_1["side"], trade_b_1["price"],
trade_b_2["side"], trade_b_2["price"]
)
pair_return = symbol_a_return + symbol_b_return
# Create round-trip records for both symbols
funding_per_position = self.config_.get_value("funding_per_pair", 10000) / 2
# Symbol A round-trip
day_roundtrips.append({
"symbol": trade_a_1["symbol"],
"open_side": trade_a_1["side"],
"open_price": trade_a_1["price"],
"open_time": trade_a_1["time"],
"close_side": trade_a_2["side"],
"close_price": trade_a_2["price"],
"close_time": trade_a_2["time"],
"symbol_return": symbol_a_return,
"pair_return": pair_return,
"shares": funding_per_position / trade_a_1["price"],
"close_condition": trade_a_2.get("status", "UNKNOWN"),
"open_disequilibrium": trade_a_1.get("disequilibrium"),
"close_disequilibrium": trade_a_2.get("disequilibrium"),
})
# Symbol B round-trip
day_roundtrips.append({
"symbol": trade_b_1["symbol"],
"open_side": trade_b_1["side"],
"open_price": trade_b_1["price"],
"open_time": trade_b_1["time"],
"close_side": trade_b_2["side"],
"close_price": trade_b_2["price"],
"close_time": trade_b_2["time"],
"symbol_return": symbol_b_return,
"pair_return": pair_return,
"shares": funding_per_position / trade_b_1["price"],
"close_condition": trade_b_2.get("status", "UNKNOWN"),
"open_disequilibrium": trade_b_1.get("disequilibrium"),
"close_disequilibrium": trade_b_2.get("disequilibrium"),
})
if day_roundtrips:
self.symbol_roundtrip_trades_[day] = day_roundtrips
def print_returns_by_day(self) -> None:
"""
Print detailed return information for each day, grouped by day.
Shows individual symbol round-trips and daily totals.
"""
print("\n====== PAIR RESEARCH RETURNS BY DAY ======")
total_return_all_days = 0.0
for day, day_trades in sorted(self.symbol_roundtrip_trades_.items()):
print(f"\n--- {day} ---")
day_total_return = 0.0
pair_returns = []
# Group trades by pair (every 2 trades form a pair)
for idx in range(0, len(day_trades), 2):
if idx + 1 < len(day_trades):
trade_a = day_trades[idx]
trade_b = day_trades[idx + 1]
# Print individual symbol results
print(f" {trade_a['open_time'].time()}-{trade_a['close_time'].time()}")
print(f" {trade_a['symbol']}: {trade_a['open_side']} @ ${trade_a['open_price']:.2f} → "
f"{trade_a['close_side']} @ ${trade_a['close_price']:.2f} | "
f"Return: {trade_a['symbol_return']:+.2f}% | Shares: {trade_a['shares']:.2f}")
print(f" {trade_b['symbol']}: {trade_b['open_side']} @ ${trade_b['open_price']:.2f} → "
f"{trade_b['close_side']} @ ${trade_b['close_price']:.2f} | "
f"Return: {trade_b['symbol_return']:+.2f}% | Shares: {trade_b['shares']:.2f}")
# Show disequilibrium info if available
if trade_a.get('open_disequilibrium') is not None:
print(f" Disequilibrium: Open: {trade_a['open_disequilibrium']:.4f}, "
f"Close: {trade_a['close_disequilibrium']:.4f}")
pair_return = trade_a['pair_return']
print(f" Pair Return: {pair_return:+.2f}% | Close Condition: {trade_a['close_condition']}")
print()
pair_returns.append(pair_return)
day_total_return += pair_return
print(f" Day Total Return: {day_total_return:+.2f}% ({len(pair_returns)} pairs)")
total_return_all_days += day_total_return
print(f"\n====== TOTAL RETURN ACROSS ALL DAYS ======")
print(f"Total Return: {total_return_all_days:+.2f}%")
print(f"Total Days: {len(self.symbol_roundtrip_trades_)}")
if len(self.symbol_roundtrip_trades_) > 0:
print(f"Average Daily Return: {total_return_all_days / len(self.symbol_roundtrip_trades_):+.2f}%")
def get_return_summary(self) -> Dict[str, Any]:
"""
Get a summary of returns across all days.
Returns a dictionary with key metrics.
"""
if len(self.symbol_roundtrip_trades_) == 0:
return {
"total_return": 0.0,
"total_days": 0,
"total_pairs": 0,
"average_daily_return": 0.0,
"best_day": None,
"worst_day": None,
"daily_returns": {}
}
daily_returns = {}
total_return = 0.0
total_pairs = 0
for day, day_trades in self.symbol_roundtrip_trades_.items():
day_return = 0.0
day_pairs = len(day_trades) // 2 # Each pair has 2 symbol trades
for trade in day_trades:
day_return += trade['symbol_return']
daily_returns[day] = {
"return": day_return,
"pairs": day_pairs
}
total_return += day_return
total_pairs += day_pairs
best_day = max(daily_returns.items(), key=lambda x: x[1]["return"]) if daily_returns else None
worst_day = min(daily_returns.items(), key=lambda x: x[1]["return"]) if daily_returns else None
return {
"total_return": total_return,
"total_days": len(self.symbol_roundtrip_trades_),
"total_pairs": total_pairs,
"average_daily_return": total_return / len(self.symbol_roundtrip_trades_) if self.symbol_roundtrip_trades_ else 0.0,
"best_day": best_day,
"worst_day": worst_day,
"daily_returns": daily_returns
}
def print_grand_totals(self) -> None:
"""Print grand totals for the single pair analysis."""
summary = self.get_return_summary()
print(f"\n====== PAIR RESEARCH GRAND TOTALS ======")
print('---')
print(f"Total Return: {summary['total_return']:+.2f}%")
print('---')
print(f"Total Days Traded: {summary['total_days']}")
print(f"Total Open-Close Actions: {summary['total_pairs']}")
print(f"Total Trades: 4 * {summary['total_pairs']} = {4 * summary['total_pairs']}")
if summary['total_days'] > 0:
print(f"Average Daily Return: {summary['average_daily_return']:+.2f}%")
if summary['best_day']:
best_day, best_data = summary['best_day']
print(f"Best Day: {best_day} ({best_data['return']:+.2f}%)")
if summary['worst_day']:
worst_day, worst_data = summary['worst_day']
print(f"Worst Day: {worst_day} ({worst_data['return']:+.2f}%)")
# Update the total_realized_pnl for backward compatibility
self.total_realized_pnl = summary['total_return']
def analyze_pair_performance(self) -> None:
"""
Main method to perform comprehensive pair research analysis.
Extracts round-trip trades, calculates returns, groups by day, and prints results.
"""
print(f"\n{'='*60}")
print(f"PAIR RESEARCH PERFORMANCE ANALYSIS")
print(f"{'='*60}")
self.calculate_returns()
self.print_returns_by_day()
self.print_outstanding_positions()
self._print_additional_metrics()
self.print_grand_totals()
def _print_additional_metrics(self) -> None:
"""Print additional performance metrics."""
summary = self.get_return_summary()
if summary['total_days'] == 0:
return
print(f"\n====== ADDITIONAL METRICS ======")
# Calculate win rate
winning_days = sum(1 for day_data in summary['daily_returns'].values() if day_data['return'] > 0)
win_rate = (winning_days / summary['total_days']) * 100
print(f"Winning Days: {winning_days}/{summary['total_days']} ({win_rate:.1f}%)")
# Calculate average trade return
if summary['total_pairs'] > 0:
# Each pair has 2 symbol trades, so total symbol trades = total_pairs * 2
total_symbol_trades = summary['total_pairs'] * 2
avg_symbol_return = summary['total_return'] / total_symbol_trades
print(f"Average Symbol Return: {avg_symbol_return:+.2f}%")
avg_pair_return = summary['total_return'] / summary['total_pairs'] / 2 # Divide by 2 since we sum both symbols
print(f"Average Pair Return: {avg_pair_return:+.2f}%")
# Show daily return distribution
daily_returns_list = [data['return'] for data in summary['daily_returns'].values()]
if daily_returns_list:
print(f"Daily Return Range: {min(daily_returns_list):+.2f}% to {max(daily_returns_list):+.2f}%")
def print_outstanding_positions(self) -> None:
"""Print outstanding positions for the single pair."""
all_positions: List[OutstandingPositionT] = self.outstanding_positions()
if not all_positions:
print("\n====== NO OUTSTANDING POSITIONS ======")
return
print(f"\n====== OUTSTANDING POSITIONS ======")
print(f"{'Symbol':<10} {'Side':<4} {'Shares':<10} {'Open $':<8} {'Current $':<10} {'Value $':<12}")
print("-" * 70)
total_value = 0.0
for pos in all_positions:
current_value = pos.get("last_value", 0.0)
print(f"{pos['symbol']:<10} {pos['open_side']:<4} {pos['shares']:<10.2f} "
f"{pos['open_px']:<8.2f} {pos['last_px']:<10.2f} {current_value:<12.2f}")
total_value += current_value
print("-" * 70)
print(f"{'TOTAL VALUE':<60} ${total_value:<12.2f}")
def get_total_realized_pnl(self) -> float:
"""Get total realized PnL."""
return self.total_realized_pnl
__SAV__/lib/pt_strategy/trading_pair.py
+226
View File
@@ -0,0 +1,226 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import Any, Dict, List
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import DataWindowParams
from pairs_trading.lib.pt_strategy.prediction import Prediction
class PairState(Enum):
INITIAL = 1
OPEN = 2
CLOSE = 3
CLOSE_POSITION = 4
CLOSE_STOP_LOSS = 5
CLOSE_STOP_PROFIT = 6
class TradingPair(NamedObject, ABC):
config_: Config
model_: Any # "PairsTradingModel"
market_data_: pd.DataFrame
user_data_: Dict[str, Any]
stat_model_price_: str
instruments_: List[ExchangeInstrument]
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument],
):
from pairs_trading.lib.pt_strategy.pt_model import PairsTradingModel
self.config_ = config
self.model_ = PairsTradingModel.create(config)
self.user_data_ = {}
self.instruments_ = instruments
self.instruments_[0].user_data_["symbol"] = instruments[0].instrument_id().split("-", 1)[1]
self.instruments_[1].user_data_["symbol"] = instruments[1].instrument_id().split("-", 1)[1]
self.stat_model_price_ = config.get_value("model/stat_model_price")
def run(self, market_data: pd.DataFrame, data_params: DataWindowParams) -> Prediction: # type: ignore[assignment]
self.market_data_ = market_data[
data_params.training_start_index_ : data_params.training_start_index_ + data_params.training_size_
]
return self.model_.predict(pair=self)
def colnames(self) -> List[str]:
return [
f"{self.stat_model_price_}_{self.symbol_a()}",
f"{self.stat_model_price_}_{self.symbol_b()}",
]
def symbol_a(self) -> str:
return self.get_instrument_a().user_data_["symbol"]
def symbol_b(self) -> str:
return self.get_instrument_b().user_data_["symbol"]
def get_instrument_a(self) -> ExchangeInstrument:
return self.instruments_[0]
def get_instrument_b(self) -> ExchangeInstrument:
return self.instruments_[1]
def __repr__(self) -> str:
return (
f"{self.__class__.__name__}:"
f" symbol_a={self.symbol_a()},"
f" symbol_b={self.symbol_b()},"
f" model={self.model_.__class__.__name__}"
)
class ResearchTradingPair(TradingPair):
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument],
):
assert len(instruments) == 2, "Trading pair must have exactly 2 instruments"
super().__init__(config=config, instruments=instruments)
self.user_data_ = {
"state": PairState.INITIAL,
}
def is_closed(self) -> bool:
return self.user_data_["state"] in [
PairState.CLOSE,
PairState.CLOSE_POSITION,
PairState.CLOSE_STOP_LOSS,
PairState.CLOSE_STOP_PROFIT,
]
def is_open(self) -> bool:
return not self.is_closed()
def exec_prices_colnames(self) -> List[str]:
return [
f"exec_price_{self.symbol_a()}",
f"exec_price_{self.symbol_b()}",
]
def to_stop_close_conditions(self, predicted_row: pd.Series) -> bool:
config = self.config_
if (
not config.key_exists("stop_close_conditions")
or config.get_value("stop_close_conditions") is None
):
return False
if "profit" in config.get_value("stop_close_conditions"):
current_return = self._current_return(predicted_row)
#
# print(f"time={predicted_row['tstamp']} current_return={current_return}")
#
if current_return >= config.get_value("stop_close_conditions")["profit"]:
print(f"STOP PROFIT: {current_return}")
self.user_data_["stop_close_state"] = PairState.CLOSE_STOP_PROFIT
return True
if "loss" in config.get_value("stop_close_conditions"):
if current_return <= config.get_value("stop_close_conditions")["loss"]:
print(f"STOP LOSS: {current_return}")
self.user_data_["stop_close_state"] = PairState.CLOSE_STOP_LOSS
return True
return False
def _current_return(self, predicted_row: pd.Series) -> float:
if "open_trades" in self.user_data_:
open_trades = self.user_data_["open_trades"]
if len(open_trades) == 0:
return 0.0
def _single_instrument_return(symbol: str) -> float:
instrument_open_trades = open_trades[open_trades["symbol"] == symbol]
instrument_open_price = instrument_open_trades["price"].iloc[0]
sign = -1 if instrument_open_trades["side"].iloc[0] == "SELL" else 1
instrument_price = predicted_row[f"{self.stat_model_price_}_{symbol}"]
instrument_return = (
sign
* (instrument_price - instrument_open_price)
/ instrument_open_price
)
return float(instrument_return) * 100.0
instrument_a_return = _single_instrument_return(self.symbol_a())
instrument_b_return = _single_instrument_return(self.symbol_b())
return instrument_a_return + instrument_b_return
return 0.0
def on_open_trades(self, trades: pd.DataFrame) -> None:
if "close_trades" in self.user_data_:
del self.user_data_["close_trades"]
self.user_data_["open_trades"] = trades
def on_close_trades(self, trades: pd.DataFrame) -> None:
del self.user_data_["open_trades"]
self.user_data_["close_trades"] = trades
def add_outstanding_position(
self,
symbol: str,
open_side: str,
open_px: float,
open_tstamp: datetime,
last_mkt_data_row: pd.Series,
) -> None:
assert symbol in [
self.symbol_a(),
self.symbol_b(),
], "Symbol must be one of the pair's symbols"
assert open_side in ["BUY", "SELL"], "Open side must be either BUY or SELL"
assert open_px > 0, "Open price must be greater than 0"
assert open_tstamp is not None, "Open timestamp must be provided"
assert last_mkt_data_row is not None, "Last market data row must be provided"
exec_prices_col_a, exec_prices_col_b = self.exec_prices_colnames()
if symbol == self.symbol_a():
last_px = last_mkt_data_row[exec_prices_col_a]
else:
last_px = last_mkt_data_row[exec_prices_col_b]
funding_per_position = self.config_.get_value("funding_per_pair") / 2
shares = funding_per_position / open_px
if open_side == "SELL":
shares = -shares
if "outstanding_positions" not in self.user_data_:
self.user_data_["outstanding_positions"] = []
self.user_data_["outstanding_positions"].append(
{
"symbol": symbol,
"open_side": open_side,
"open_px": open_px,
"shares": shares,
"open_tstamp": open_tstamp,
"last_px": last_px,
"last_tstamp": last_mkt_data_row["tstamp"],
"last_value": last_px * shares,
}
)
class LiveTradingPair(TradingPair):
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
def to_stop_close_conditions(self, predicted_row: pd.Series) -> bool:
# TODO LiveTradingPair.to_stop_close_conditions()
return False