research/stat_pairs_backtest
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new purpose

Browse Source
This commit is contained in:
Oleg Sheynin
2026-07-24 22:40:34 +00:00
parent dc38176529
commit 8ccebf81f5
34 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__SAV__/lib/live/mkt_data_client.py
+277
View File
@@ -0,0 +1,277 @@
from __future__ import annotations
import asyncio
from typing import Dict, Any, List, Optional, Set
import requests
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.logger import Log
from cvttpy_tools.base.config import Config
from cvttpy_tools.base.timer import Timer
from cvttpy_tools.base.timeutils import NanosT, current_seconds
from cvttpy_tools.settings.cvtt_types import InstrumentIdT, IntervalSecT
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
from cvttpy_trading.trading.accounting.exch_account import ExchangeAccountNameT
from cvttpy_trading.trading.mkt_data.md_summary import MdTradesAggregate, MdSummary, MdSummaryCallbackT
from cvttpy_trading.trading.exchange_config import ExchangeAccounts
# ---
from pairs_trading.lib.live.rest import RESTSender
# class MdSummary(HistMdBar):
# def __init__(
# self,
# ts_ns: int,
# open: float,
# high: float,
# low: float,
# close: float,
# volume: float,
# vwap: float,
# num_trades: int,
# ):
# super().__init__(ts=ts_ns)
# self.open_ = open
# self.high_ = high
# self.low_ = low
# self.close_ = close
# self.volume_ = volume
# self.vwap_ = vwap
# self.num_trades_ = num_trades
# @classmethod
# def from_REST_response(cls, response: requests.Response) -> List[MdSummary]:
# res: List[MdSummary] = []
# jresp = response.json()
# hist_data = jresp.get("historical_data", [])
# for hd in hist_data:
# res.append(
# MdSummary(
# ts_ns=hd["time_ns"],
# open=hd["open"],
# high=hd["high"],
# low=hd["low"],
# close=hd["close"],
# volume=hd["volume"],
# vwap=hd["vwap"],
# num_trades=hd["num_trades"],
# )
# )
# return res
# def create_md_trades_aggregate(
# self,
# exch_acct: ExchangeAccountNameT,
# exch_inst: ExchangeInstrument,
# interval_sec: IntervalSecT,
# ) -> MdTradesAggregate:
# res = MdTradesAggregate(
# exch_acct=exch_acct,
# exch_inst=exch_inst,
# interval_ns=interval_sec * NanoPerSec,
# )
# res.set(mdbar=self)
# return res
# MdSummaryCallbackT = Callable[[List[MdTradesAggregate]], Coroutine]
class MdSummaryCollector(NamedObject):
sender_: RESTSender
exch_acct_: ExchangeAccountNameT
exch_inst_: ExchangeInstrument
interval_sec_: IntervalSecT
history_depth_sec_: IntervalSecT
history_: List[MdTradesAggregate]
callbacks_: List[MdSummaryCallbackT]
timer_: Optional[Timer]
def __init__(
self,
sender: RESTSender,
exch_acct: ExchangeAccountNameT,
instrument_id: InstrumentIdT,
interval_sec: IntervalSecT,
history_depth_sec: IntervalSecT,
) -> None:
self.sender_ = sender
self.exch_acct_ = exch_acct
exch_inst = ExchangeAccounts.instance().get_exchange_instrument(
exch_acct=exch_acct, instrument_id=instrument_id
)
assert exch_inst is not None, f"Unable to find Exchange instrument for {exch_acct}/{instrument_id}"
self.exch_inst_ = exch_inst
self.interval_sec_ = interval_sec
self.history_depth_sec_ = history_depth_sec
self.history_ = []
self.callbacks_ = []
self.timer_ = None
def add_callback(self, cb: MdSummaryCallbackT) -> None:
self.callbacks_.append(cb)
def __hash__(self):
return hash(
(
self.exch_acct_,
self.exch_inst_.instrument_id(),
self.interval_sec_,
self.history_depth_sec_,
)
)
def rqst_data(self) -> Dict[str, Any]:
return {
"exch_acct": self.exch_acct_,
"instrument_id": self.exch_inst_.instrument_id(),
"interval_sec": self.interval_sec_,
"history_depth_sec": self.history_depth_sec_,
}
def get_history(self) -> List[MdSummary]:
response: requests.Response = self.sender_.send_post(
endpoint="md_summary", post_body=self.rqst_data()
)
if response.status_code not in (200, 201):
Log.error(
f"{self.fname()}: Received error: {response.status_code} - {response.text}"
)
return []
return MdSummary.from_REST_response(response=response)
def get_last(self) -> Optional[MdSummary]:
Log.info(f"{self.fname()}: for {self.exch_inst_.details_short()}")
rqst_data = self.rqst_data()
rqst_data["history_depth_sec"] = self.interval_sec_ * 2
response: requests.Response = self.sender_.send_post(
endpoint="md_summary", post_body=rqst_data
)
if response.status_code not in (200, 201):
Log.error(
f"{self.fname()}: Received error: {response.status_code} - {response.text}"
)
return None
res = MdSummary.from_REST_response(response=response)
Log.info(f"DEBUG *** {self.exch_inst_.base_asset_id_}: {res[-1].tstamp_}")
return None if len(res) == 0 else res[-1]
def is_empty(self) -> bool:
return len(self.history_) == 0
async def start(self) -> None:
if self.timer_:
Log.error(f"{self.fname()}: Timer is already started")
return
mdsum_hist = self.get_history()
self.history_ = [
mdsum.create_md_trades_aggregate(
exch_acct=self.exch_acct_,
exch_inst=self.exch_inst_,
interval_sec=self.interval_sec_,
)
for mdsum in mdsum_hist
]
await self.run_callbacks()
self.set_timer()
def set_timer(self):
if self.timer_:
self.timer_.cancel()
start_in = self.next_load_time() - current_seconds()
self.timer_ = Timer(
start_in_sec=start_in,
func=self._load_new,
)
Log.info(f"{self.fname()} Timer for {self.exch_inst_.details_short()} is set to run in {start_in} sec")
def next_load_time(self) -> NanosT:
ALLOW_LAG_SEC = 1
curr_sec = int(current_seconds())
return (curr_sec - curr_sec % self.interval_sec_) + self.interval_sec_ + ALLOW_LAG_SEC
async def _load_new(self) -> None:
last: Optional[MdSummary] = self.get_last()
if not last:
Log.warning(f"{self.fname()}: did not get last update")
elif not self.is_empty() and last.ts_ns_ <= self.history_[-1].aggr_time_ns_:
Log.info(
f"{self.fname()}: Received {last}. Already Have: {self.history_[-1]}"
)
else:
self.history_.append(last.create_md_trades_aggregate(exch_acct=self.exch_acct_, exch_inst=self.exch_inst_, interval_sec=self.interval_sec_))
await self.run_callbacks()
self.set_timer()
async def run_callbacks(self) -> None:
[await cb(self.history_) for cb in self.callbacks_]
def stop(self) -> None:
if self.timer_:
self.timer_.cancel()
self.timer_ = None
class CvttRestMktDataClient(NamedObject):
config_: Config
sender_: RESTSender
collectors_: Set[MdSummaryCollector]
def __init__(self, config: Config) -> None:
self.config_ = config
base_url = self.config_.get_value("cvtt_base_url", default="")
assert base_url
self.sender_ = RESTSender(base_url=base_url)
self.collectors_ = set()
async def add_subscription(
self,
exch_acct: ExchangeAccountNameT,
instrument_id: InstrumentIdT,
interval_sec: IntervalSecT,
history_depth_sec: IntervalSecT,
callback: MdSummaryCallbackT,
) -> None:
mdsc = MdSummaryCollector(
sender=self.sender_,
exch_acct=exch_acct,
instrument_id=instrument_id,
interval_sec=interval_sec,
history_depth_sec=history_depth_sec,
)
mdsc.add_callback(callback)
self.collectors_.add(mdsc)
await mdsc.start()
if __name__ == "__main__":
config = Config(json_src={"cvtt_base_url": "http://cvtt-tester-01.cvtt.vpn:23456"})
# config = Config(json_src={"cvtt_base_url": "http://dev-server-02.cvtt.vpn:23456"})
async def _calback(history: List[MdTradesAggregate]) -> None:
Log.info(
f"MdSummary Hist Length is {len(history)}. Last summary: {history[-1] if len(history) > 0 else '[]'}"
)
async def __run() -> None:
Log.info("Starting...")
cvtt_client = CvttRestMktDataClient(config)
await cvtt_client.add_subscription(
exch_acct="COINBASE_AT",
instrument_id="PAIR-BTC-USD",
interval_sec=60,
history_depth_sec=24 * 3600,
callback=_calback,
)
while True:
await asyncio.sleep(5)
asyncio.run(__run())
pass
__SAV__/lib/live/rest.py
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from typing import Dict, Optional
import time
import requests
from cvttpy_tools.base.base import NamedObject
class RESTSender(NamedObject):
# Synchronous request sernder
session_: requests.Session
base_url_: str
def __init__(self, base_url: str) -> None:
self.base_url_ = base_url
self.session_ = requests.Session()
def is_ready(self) -> bool:
"""Checks if the server is up and responding"""
url = f"{self.base_url_}/ping"
try:
response = self.session_.get(url)
response.raise_for_status()
return True
except requests.exceptions.RequestException:
return False
def send_post(
self, endpoint: str, post_body: Dict, headers: Optional[Dict[str, str]] = None
) -> requests.Response:
if not headers:
headers = {"Content-Type": "application/json"}
url = f"{self.base_url_}/{endpoint}"
try:
return self.session_.request(
method="POST",
url=url,
json=post_body,
headers=headers,
)
except requests.exceptions.RequestException as excpt:
raise ConnectionError(
f"Failed to send status={excpt.response.status_code} {excpt.response.text}" # type: ignore
) from excpt
def send_get(
self, endpoint: str, headers: Optional[Dict[str, str]] = None
) -> requests.Response:
if not headers:
headers = {}
url = f"{self.base_url_}/{endpoint}"
try:
return self.session_.request(method="GET", url=url, headers=headers)
except requests.exceptions.RequestException as excpt:
raise ConnectionError(
f"Failed to send status={excpt.response.status_code} {excpt.response.text}" # type: ignore
) from excpt
__SAV__/lib/live/ti_sender.py
+50
View File
@@ -0,0 +1,50 @@
from enum import Enum
import requests
# import aiohttp
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.config import Config
from cvttpy_tools.base.logger import Log
# ---
from cvttpy_trading.trading.trading_instructions import TradingInstructions
# ---
from pairs_trading.apps.pair_trader import PairTrader
from pairs_trading.lib.live.rest import RESTSender
class TradingInstructionsSender(NamedObject):
config_: Config
sender_: RESTSender
pairs_trader_: PairTrader
class TradingInstType(str, Enum):
TARGET_POSITION = "TARGET_POSITION"
DIRECT_ORDER = "DIRECT_ORDER"
MARKET_MAKING = "MARKET_MAKING"
NONE = "NONE"
def __init__(self, config: Config, pairs_trader: PairTrader) -> None:
self.config_ = config
base_url = self.config_.get_value("cvtt_base_url", default="")
assert base_url
self.sender_ = RESTSender(base_url=base_url)
self.pairs_trader_ = pairs_trader
self.book_id_ = self.pairs_trader_.book_id_
assert self.book_id_, "book_id is required"
self.strategy_id_ = config.get_value("strategy_id", "")
assert self.strategy_id_, "strategy_id is required"
async def send_trading_instructions(self, ti: TradingInstructions) -> None:
Log.info(f"{self.fname()}: sending {ti=}")
response: requests.Response = self.sender_.send_post(
endpoint="trading_instructions", post_body=ti.to_dict()
)
if response.status_code not in (200, 201):
Log.error(
f"{self.fname()}: Received error: {response.status_code} - {response.text}"
)
__SAV__/lib/pt_strategy/live/live_strategy.py
+351
View File
@@ -0,0 +1,351 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.app import App
from cvttpy_tools.base.config import Config
from cvttpy_tools.settings.cvtt_types import IntervalSecT
from cvttpy_tools.base.timeutils import NanosT, SecPerHour, current_nanoseconds, NanoPerSec, format_nanos_utc
from cvttpy_tools.base.logger import Log
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
from cvttpy_trading.trading.mkt_data.md_summary import MdTradesAggregate
from cvttpy_trading.trading.trading_instructions import TradingInstructions
from cvttpy_trading.trading.trading_instructions import TargetPositionSignal
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.pt_model import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import LiveTradingPair
from pairs_trading.apps.pair_trader import PairTrader
from pairs_trading.lib.pt_strategy.pt_market_data import LiveMarketData
class PtLiveStrategy(NamedObject):
config_: Config
instruments_: List[ExchangeInstrument]
interval_sec_: IntervalSecT
history_depth_sec_: IntervalSecT
open_threshold_: float
close_threshold_: float
trading_pair_: LiveTradingPair
model_data_policy_: ModelDataPolicy
pairs_trader_: PairTrader
# for presentation: history of prediction values and trading signals
predictions_df_: pd.DataFrame
trading_signals_df_: pd.DataFrame
allowed_md_lag_sec_: int
def __init__(
self,
config: Config,
pairs_trader: PairTrader,
):
self.config_ = config
self.pairs_trader_ = pairs_trader
self.trading_pair_ = LiveTradingPair(
config=config,
instruments=self.pairs_trader_.instruments_,
)
self.model_data_policy_ = ModelDataPolicy.create(
self.config_,
is_real_time=True,
pair=self.trading_pair_,
)
assert (
self.model_data_policy_ is not None
), f"{self.fname()}: Unable to create ModelDataPolicy"
self.predictions_df_ = pd.DataFrame()
self.trading_signals_df_ = pd.DataFrame()
self.instruments_ = self.pairs_trader_.instruments_
App.instance().add_call(
stage=App.Stage.Config, func=self._on_config(), can_run_now=True
)
async def _on_config(self) -> None:
self.interval_sec_ = self.config_.get_value("interval_sec", 0)
assert self.interval_sec_ > 0, "interval_sec cannot be 0"
self.history_depth_sec_ = (
self.config_.get_value("history_depth_hours", 0) * SecPerHour
)
assert self.history_depth_sec_ > 0, "history_depth_hours cannot be 0"
self.allowed_md_lag_sec_ = self.config_.get_value("allowed_md_lag_sec", 3)
self.open_threshold_ = self.config_.get_value(
"model/disequilibrium/open_trshld", 0.0
)
self.close_threshold_ = self.config_.get_value(
"model/disequilibrium/close_trshld", 0.0
)
assert (
self.open_threshold_ > 0
), "disequilibrium/open_trshld must be greater than 0"
assert (
self.close_threshold_ > 0
), "disequilibrium/close_trshld must be greater than 0"
await self.pairs_trader_.subscribe_md()
def __repr__(self) -> str:
return f"{self.classname()}: trading_pair={self.trading_pair_}, mdp={self.model_data_policy_.__class__.__name__}, "
async def on_mkt_data_hist_snapshot(
self, hist_aggr: List[MdTradesAggregate]
) -> None:
if not self._is_md_actual(hist_aggr=hist_aggr):
return
market_data_df: pd.DataFrame = self._create_md_df(hist_aggr=hist_aggr)
if len(market_data_df) == 0:
Log.warning(f"{self.fname()} Unable to create market data df")
return
self.trading_pair_.market_data_ = market_data_df
Log.info(f"{self.fname()}: Running prediction for pair: {self.trading_pair_}")
prediction = self.trading_pair_.run(
market_data_df, self.model_data_policy_.advance()
)
self.predictions_df_ = pd.concat(
[self.predictions_df_, prediction.to_df()], ignore_index=True
)
trading_instructions: List[TradingInstructions] = (
self._create_trading_instructions(
prediction=prediction, last_row=market_data_df.iloc[-1]
)
)
if trading_instructions is not None:
await self._send_trading_instructions(trading_instructions)
def _is_md_actual(self, hist_aggr: List[MdTradesAggregate]) -> bool:
if len(hist_aggr) == 0:
Log.warning(f"{self.fname()} list of aggregates IS EMPTY")
return False
curr_ns = current_nanoseconds()
# MAYBE check market data length
# at 18:05:01 we should see data for 18:04:00
lag_sec = (curr_ns - hist_aggr[-1].aggr_time_ns_) / NanoPerSec - self.interval_sec()
if lag_sec > self.allowed_md_lag_sec_:
Log.warning(
f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
f" Lagging {int(lag_sec)} > {self.allowed_md_lag_sec_} seconds:"
f"\n{len(hist_aggr)} records"
f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
)
return False
else:
Log.info(
f"{self.fname()} {hist_aggr[-1].exch_inst_.details_short()}"
f" Lag {int(lag_sec)} <= {self.allowed_md_lag_sec_} seconds"
f"\n{len(hist_aggr)} records"
f"\n{hist_aggr[-1].exch_inst_.base_asset_id_}: {hist_aggr[-1].tstamp()}"
f"\n{hist_aggr[-2].exch_inst_.base_asset_id_}: {hist_aggr[-2].tstamp()}"
)
return True
def _create_md_df(self, hist_aggr: List[MdTradesAggregate]) -> pd.DataFrame:
"""
tstamp time_ns symbol open high low close volume num_trades vwap
0 2025-09-10 11:30:00 1757503800000000000 ADA-USDT 0.8750 0.8750 0.8743 0.8743 50710.500 0 0.874489
1 2025-09-10 11:30:00 1757503800000000000 SOL-USDT 219.9700 219.9800 219.6600 219.7000 2648.582 0 219.787847
2 2025-09-10 11:31:00 1757503860000000000 SOL-USDT 219.7000 219.7300 219.6200 219.6200 1134.886 0 219.663460
3 2025-09-10 11:31:00 1757503860000000000 ADA-USDT 0.8743 0.8745 0.8741 0.8741 10696.400 0 0.874234
4 2025-09-10 11:32:00 1757503920000000000 ADA-USDT 0.8742 0.8742 0.8739 0.8740 18546.900 0 0.874037
"""
rows: List[Dict[str, Any]] = []
for aggr in hist_aggr:
exch_inst = aggr.exch_inst_
rows.append(
{
# convert nanoseconds → tz-aware pandas timestamp
"tstamp": pd.to_datetime(aggr.aggr_time_ns_, unit="ns", utc=True),
"time_ns": aggr.aggr_time_ns_,
"symbol": exch_inst.instrument_id().split("-", 1)[1],
"exchange_id": exch_inst.exchange_id_,
"instrument_id": exch_inst.instrument_id(),
"open": exch_inst.get_price(aggr.open_),
"high": exch_inst.get_price(aggr.high_),
"low": exch_inst.get_price(aggr.low_),
"close": exch_inst.get_price(aggr.close_),
"volume": exch_inst.get_quantity(aggr.volume_),
"num_trades": aggr.num_trades_,
"vwap": exch_inst.get_price(aggr.vwap_),
}
)
source_md_df = pd.DataFrame(
rows,
columns=[
"tstamp",
"time_ns",
"symbol",
"exchange_id",
"instrument_id",
"open",
"high",
"low",
"close",
"volume",
"num_trades",
"vwap",
],
)
# automatic sorting
source_md_df.sort_values(
by=["time_ns", "symbol"],
ascending=True,
inplace=True,
kind="mergesort", # stable sort
)
source_md_df.reset_index(drop=True, inplace=True)
pt_mkt_data = LiveMarketData(config=self.config_, instruments=self.instruments_)
pt_mkt_data.origin_mkt_data_df_ = source_md_df
pt_mkt_data.set_market_data()
return pt_mkt_data.market_data_df_
def interval_sec(self) -> IntervalSecT:
return self.interval_sec_
def history_depth_sec(self) -> IntervalSecT:
return self.history_depth_sec_
async def _send_trading_instructions(
self, trading_instructions: List[TradingInstructions]
) -> None:
for ti in trading_instructions:
Log.info(f"{self.fname()} Sending trading instructions {ti}")
await self.pairs_trader_.ti_sender_.send_trading_instructions(ti)
def _create_trading_instructions(
self, prediction: Prediction, last_row: pd.Series
) -> List[TradingInstructions]:
trd_instructions: List[TradingInstructions] = []
pair = self.trading_pair_
scaled_disequilibrium = prediction.scaled_disequilibrium_
abs_scaled_disequilibrium = abs(scaled_disequilibrium)
if abs_scaled_disequilibrium >= self.open_threshold_:
trd_instructions = self._create_open_trade_instructions(
pair, row=last_row, prediction=prediction
)
elif abs_scaled_disequilibrium <= self.close_threshold_ or pair.to_stop_close_conditions(predicted_row=last_row):
trd_instructions = self._create_close_trade_instructions(
pair, row=last_row # , prediction=prediction
)
return trd_instructions
def _strength(self, scaled_disequilibrium: float) -> float:
# TODO PtLiveStrategy._strength()
return 1.0
def _create_open_trade_instructions(
self, pair: LiveTradingPair, row: pd.Series, prediction: Prediction
) -> List[TradingInstructions]:
diseqlbrm = prediction.disequilibrium_
scaled_disequilibrium = prediction.scaled_disequilibrium_
if diseqlbrm > 0:
side_a = -1
side_b = 1
else:
side_a = 1
side_b = -1
ti_a: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=side_a * self._strength(scaled_disequilibrium),
exchange_id=pair.get_instrument_a().exchange_id_,
base_asset=pair.get_instrument_a().base_asset_id_,
quote_asset=pair.get_instrument_a().quote_asset_id_,
user_data={}
),
)
if not ti_a:
return []
ti_b: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=side_b * self._strength(scaled_disequilibrium),
exchange_id=pair.get_instrument_b().exchange_id_,
base_asset=pair.get_instrument_b().base_asset_id_,
quote_asset=pair.get_instrument_b().quote_asset_id_,
user_data={}
),
)
if not ti_b:
return []
return [ti_a, ti_b]
def _create_close_trade_instructions(
self, pair: LiveTradingPair, row: pd.Series
) -> List[TradingInstructions]:
ti_a: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=0,
exchange_id=pair.get_instrument_a().exchange_id_,
base_asset=pair.get_instrument_a().base_asset_id_,
quote_asset=pair.get_instrument_a().quote_asset_id_,
user_data={}
),
)
if not ti_a:
return []
ti_b: Optional[TradingInstructions] = TradingInstructions(
book=self.pairs_trader_.book_id_,
strategy_id=self.__class__.__name__,
ti_type=TradingInstructions.Type.TARGET_POSITION,
issued_ts_ns=current_nanoseconds(),
data=TargetPositionSignal(
strength=0,
exchange_id=pair.get_instrument_b().exchange_id_,
base_asset=pair.get_instrument_b().base_asset_id_,
quote_asset=pair.get_instrument_b().quote_asset_id_,
user_data={}
),
)
if not ti_b:
return []
return [ti_a, ti_b]
__SAV__/lib/pt_strategy/model_data_policy.py
+253
View File
@@ -0,0 +1,253 @@
from __future__ import annotations
import copy
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Dict, Optional, cast
import numpy as np
import pandas as pd
from cvttpy_tools.base.config import Config
@dataclass
class DataWindowParams:
training_size_: int
training_start_index_: int
class ModelDataPolicy(ABC):
config_: Config
current_data_params_: DataWindowParams
count_: int
is_real_time_: bool
def __init__(self, config: Config, *args: Any, **kwargs: Any):
self.config_ = config
self.current_data_params_ = DataWindowParams(
training_size_=config.get_value("model/training_size", 120),
training_start_index_=0,
)
self.count_ = 0
self.is_real_time_ = kwargs.get("is_real_time", False)
@abstractmethod
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
self.count_ += 1
if not self.is_real_time_:
print(self.count_, end="\r")
return self.current_data_params_
@staticmethod
def create(config: Config, *args: Any, **kwargs: Any) -> ModelDataPolicy:
import importlib
model_data_policy_class_name = config.get_value("model/model_data_policy_class", None)
assert model_data_policy_class_name is not None
module_name, class_name = model_data_policy_class_name.rsplit(".", 1)
module = importlib.import_module(module_name)
model_training_data_policy_object = getattr(module, class_name)(
config=config, *args, **kwargs
)
return cast(ModelDataPolicy, model_training_data_policy_object)
class RollingWindowDataPolicy(ModelDataPolicy):
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
self.count_ = 1
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
super().advance(mkt_data_df)
if self.is_real_time_:
self.current_data_params_.training_start_index_ = 0
if mkt_data_df and len(mkt_data_df) > self.curren_data_params_.training_size_:
self.current_data_params_.training_start_index_ = -self.curren_data_params_.training_size_
else:
self.current_data_params_.training_start_index_ += 1
return self.current_data_params_
class OptimizedWndDataPolicy(ModelDataPolicy, ABC):
mkt_data_df_: pd.DataFrame
pair_: TradingPair # type: ignore
min_training_size_: int
max_training_size_: int
end_index_: int
prices_a_: np.ndarray
prices_b_: np.ndarray
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
assert (
kwargs.get("pair") is not None
), "pair must be provided"
assert (config.key_exists("model/max_training_size") and config.key_exists("model/min_training_size")
), "min_training_size and max_training_size must be provided"
self.min_training_size_ = cast(int, config.get_value("model/min_training_size"))
self.max_training_size_ = cast(int, config.get_value("model/max_training_size"))
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
self.pair_ = cast(TradingPair, kwargs.get("pair"))
if "mkt_data" in kwargs:
self.mkt_data_df_ = cast(pd.DataFrame, kwargs.get("mkt_data"))
col_a, col_b = self.pair_.colnames()
self.prices_a_ = np.array(self.mkt_data_df_[col_a])
self.prices_b_ = np.array(self.mkt_data_df_[col_b])
assert self.min_training_size_ < self.max_training_size_
def advance(self, mkt_data_df: Optional[pd.DataFrame] = None) -> DataWindowParams:
super().advance(mkt_data_df)
if mkt_data_df is not None:
self.mkt_data_df_ = mkt_data_df
if self.is_real_time_:
self.end_index_ = len(self.mkt_data_df_) - 1
else:
self.end_index_ = self.current_data_params_.training_start_index_ + self.max_training_size_
if self.end_index_ > len(self.mkt_data_df_) - 1:
self.end_index_ = len(self.mkt_data_df_) - 1
self.current_data_params_.training_start_index_ = self.end_index_ - self.max_training_size_
if self.current_data_params_.training_start_index_ < 0:
self.current_data_params_.training_start_index_ = 0
col_a, col_b = self.pair_.colnames()
self.prices_a_ = np.array(self.mkt_data_df_[col_a])
self.prices_b_ = np.array(self.mkt_data_df_[col_b])
self.current_data_params_ = self.optimize_window_size()
return self.current_data_params_
@abstractmethod
def optimize_window_size(self) -> DataWindowParams:
...
class EGOptimizedWndDataPolicy(OptimizedWndDataPolicy):
'''
# Engle-Granger cointegration test
*** VERY SLOW ***
'''
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
# Run Engle-Granger cointegration test
last_pvalue = 1.0
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
from statsmodels.tsa.stattools import coint # type: ignore
start_index = self.end_index_ - trn_size
series_a = self.prices_a_[start_index : self.end_index_]
series_b = self.prices_b_[start_index : self.end_index_]
eg_pvalue = float(coint(series_a, series_b)[1])
if eg_pvalue < last_pvalue:
last_pvalue = eg_pvalue
result.training_size_ = trn_size
result.training_start_index_ = start_index
# print(
# f"*** DEBUG *** end_index={self.end_index_}, best_trn_size={self.current_data_params_.training_size}, {last_pvalue=}"
# )
return result
class ADFOptimizedWndDataPolicy(OptimizedWndDataPolicy):
# Augmented Dickey-Fuller test
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
from statsmodels.regression.linear_model import OLS
from statsmodels.tools.tools import add_constant
from statsmodels.tsa.stattools import adfuller
last_pvalue = 1.0
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
start_index = self.end_index_ - trn_size
y = self.prices_a_[start_index : self.end_index_]
x = self.prices_b_[start_index : self.end_index_]
# Add constant to x for intercept
x_with_const = add_constant(x)
# OLS regression: y = a + b*x + e
model = OLS(y, x_with_const).fit()
residuals = y - model.predict(x_with_const)
# ADF test on residuals
try:
adf_result = adfuller(residuals, maxlag=1, regression="c")
adf_pvalue = float(adf_result[1])
except Exception as e:
# Handle edge cases with exception (e.g., constant series, etc.)
adf_pvalue = 1.0
if adf_pvalue < last_pvalue:
last_pvalue = adf_pvalue
result.training_size_ = trn_size
result.training_start_index_ = start_index
# print(
# f"*** DEBUG *** end_index={self.end_index_},"
# f" best_trn_size={self.current_data_params_.training_size},"
# f" {last_pvalue=}"
# )
return result
class JohansenOptdWndDataPolicy(OptimizedWndDataPolicy):
# Johansen test
def __init__(self, config: Config, *args: Any, **kwargs: Any):
super().__init__(config, *args, **kwargs)
def optimize_window_size(self) -> DataWindowParams:
from statsmodels.tsa.vector_ar.vecm import coint_johansen
import numpy as np
best_stat = -np.inf
best_trn_size = 0
best_start_index = -1
result = copy.copy(self.current_data_params_)
for trn_size in range(self.min_training_size_, self.max_training_size_):
if self.end_index_ - trn_size < 0:
break
start_index = self.end_index_ - trn_size
series_a = self.prices_a_[start_index:self.end_index_]
series_b = self.prices_b_[start_index:self.end_index_]
# Combine into 2D matrix for Johansen test
try:
data = np.column_stack([series_a, series_b])
# Johansen test: det_order=0 (no deterministic trend), k_ar_diff=1 (lag)
res = coint_johansen(data, det_order=0, k_ar_diff=1)
# Trace statistic for cointegration rank 1
trace_stat = res.lr1[0] # test stat for rank=0 vs >=1
critical_value = res.cvt[0, 1] # 5% critical value
if trace_stat > best_stat:
best_stat = trace_stat
best_trn_size = trn_size
best_start_index = start_index
except Exception:
continue
if best_trn_size > 0:
result.training_size_ = best_trn_size
result.training_start_index_ = best_start_index
else:
print("*** WARNING: No valid cointegration window found.")
# print(
# f"*** DEBUG *** end_index={self.end_index_}, best_trn_size={best_trn_size}, trace_stat={best_stat}"
# )
return result
__SAV__/lib/pt_strategy/models.py
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
from typing import Optional
import pandas as pd
import statsmodels.api as sm
from pairs_trading.lib.pt_strategy.pt_model import PairsTradingModel, Prediction
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
class OLSModel(PairsTradingModel):
model_: Optional[sm.regression.linear_model.RegressionResultsWrapper]
pair_predict_result_: Optional[pd.DataFrame]
zscore_df_: Optional[pd.DataFrame]
def predict(self, pair: TradingPair) -> Prediction:
self.training_df_ = pair.market_data_.copy()
zscore_df = self._fit_zscore(pair=pair)
assert zscore_df is not None
# zscore is both disequilibrium and scaled_disequilibrium
self.training_df_["dis-equilibrium"] = zscore_df[0]
self.training_df_["scaled_dis-equilibrium"] = zscore_df[0]
assert zscore_df is not None
return Prediction(
tstamp=pair.market_data_.iloc[-1]["tstamp"],
disequilibrium=self.training_df_["dis-equilibrium"].iloc[-1],
scaled_disequilibrium=self.training_df_["scaled_dis-equilibrium"].iloc[-1],
)
def _fit_zscore(self, pair: TradingPair) -> pd.DataFrame:
assert self.training_df_ is not None
symbol_a_px_series = self.training_df_[pair.colnames()].iloc[:, 0]
symbol_b_px_series = self.training_df_[pair.colnames()].iloc[:, 1]
symbol_a_px_series, symbol_b_px_series = symbol_a_px_series.align(
symbol_b_px_series, axis=0
)
X = sm.add_constant(symbol_b_px_series)
self.model_ = sm.OLS(symbol_a_px_series, X).fit()
assert self.model_ is not None
# alternate way would be to use models residuals (will give identical results)
# alpha, beta = self.model_.params
# spread = symbol_a_px_series - (alpha + beta * symbol_b_px_series)
spread = self.model_.resid
return pd.DataFrame((spread - spread.mean()) / spread.std())
class VECMModel(PairsTradingModel):
def predict(self, pair: TradingPair) -> Prediction:
self.training_df_ = pair.market_data_.copy()
assert self.training_df_ is not None
vecm_fit = self._fit_VECM(pair=pair)
assert vecm_fit is not None
predicted_prices = vecm_fit.predict(steps=1)
# Convert prediction to a DataFrame for readability
predicted_df = pd.DataFrame(
predicted_prices, columns=pd.Index(pair.colnames()), dtype=float
)
disequilibrium = (predicted_df[pair.colnames()] @ vecm_fit.beta)[0][0]
scaled_disequilibrium = (disequilibrium - self.training_mu_) / self.training_std_
return Prediction(
tstamp=pair.market_data_.iloc[-1]["tstamp"],
disequilibrium=disequilibrium,
scaled_disequilibrium=scaled_disequilibrium,
)
def _fit_VECM(self, pair: TradingPair) -> VECMResults: # type: ignore
from statsmodels.tsa.vector_ar.vecm import VECM, VECMResults
vecm_df = self.training_df_[pair.colnames()].reset_index(drop=True)
vecm_model = VECM(vecm_df, coint_rank=1)
vecm_fit = vecm_model.fit()
assert vecm_fit is not None
# Check if the model converged properly
if not hasattr(vecm_fit, "beta") or vecm_fit.beta is None:
print(f"{self}: VECM model failed to converge properly")
diseq_series = self.training_df_[pair.colnames()] @ vecm_fit.beta
# print(diseq_series.shape)
self.training_mu_ = float(diseq_series[0].mean())
self.training_std_ = float(diseq_series[0].std())
self.training_df_["dis-equilibrium"] = (
self.training_df_[pair.colnames()] @ vecm_fit.beta
)
# Normalize the dis-equilibrium
self.training_df_["scaled_dis-equilibrium"] = (
diseq_series - self.training_mu_
) / self.training_std_
return vecm_fit
__SAV__/lib/pt_strategy/prediction.py
+28
View File
@@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Any, Dict
import pandas as pd
class Prediction:
tstamp_: pd.Timestamp
disequilibrium_: float
scaled_disequilibrium_: float
def __init__(self, tstamp: pd.Timestamp, disequilibrium: float, scaled_disequilibrium: float):
self.tstamp_ = tstamp
self.disequilibrium_ = disequilibrium
self.scaled_disequilibrium_ = scaled_disequilibrium
def to_dict(self) -> Dict[str, Any]:
return {
"tstamp": self.tstamp_,
"disequilibrium": self.disequilibrium_,
"signed_scaled_disequilibrium": self.scaled_disequilibrium_,
"scaled_disequilibrium": abs(self.scaled_disequilibrium_),
# "pair": self.pair_,
}
def to_df(self) -> pd.DataFrame:
return pd.DataFrame([self.to_dict()])
__SAV__/lib/pt_strategy/pt_market_data.py
+223
View File
@@ -0,0 +1,223 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.config import Config
from cvttpy_tools.settings.cvtt_types import JsonDictT
# ---
from cvttpy_trading.trading.mkt_data.md_summary import MdTradesAggregate
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.tools.data_loader import load_market_data
class PtMarketData(NamedObject, ABC):
config_: Config
origin_mkt_data_df_: pd.DataFrame
market_data_df_: pd.DataFrame
stat_model_price_: str
instruments_: List[ExchangeInstrument]
symbol_a_: str
symbol_b_: str
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
self.config_ = config
self.origin_mkt_data_df_ = pd.DataFrame()
self.market_data_df_ = pd.DataFrame()
self.stat_model_price_ = self.config_.get_value("model/stat_model_price")
self.instruments_ = instruments
assert len(self.instruments_) > 0, "No instruments found in config"
self.symbol_a_ = self.instruments_[0].instrument_id().split("-", 1)[1]
self.symbol_b_ = self.instruments_[1].instrument_id().split("-", 1)[1]
@abstractmethod
def md_columns(self) -> List[str]: ...
@abstractmethod
def rename_columns(self, symbol_df: pd.DataFrame) -> pd.DataFrame: ...
@abstractmethod
def tranform_df_target_colnames(self) -> List[str]: ...
def set_market_data(self) -> None:
self.market_data_df_ = pd.DataFrame(
self._transform_dataframe(self.origin_mkt_data_df_)[
["tstamp"] + self.tranform_df_target_colnames()
]
)
self.market_data_df_ = self.market_data_df_.dropna().reset_index(drop=True)
self.market_data_df_["tstamp"] = pd.to_datetime(self.market_data_df_["tstamp"])
self.market_data_df_ = self.market_data_df_.sort_values("tstamp")
def colnames(self) -> List[str]:
return [
f"{self.stat_model_price_}_{self.symbol_a_}",
f"{self.stat_model_price_}_{self.symbol_b_}",
]
def _transform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
df_selected: pd.DataFrame = pd.DataFrame(df[self.md_columns()])
result_df = (
pd.DataFrame(df_selected["tstamp"]).drop_duplicates().reset_index(drop=True)
)
# For each unique symbol, add a corresponding stat_model_price column
symbols = df_selected["symbol"].unique()
for symbol in symbols:
# Filter rows for this symbol
df_symbol = df_selected[df_selected["symbol"] == symbol].reset_index(
drop=True
)
# Create column name like "close-COIN"
temp_df: pd.DataFrame = self.rename_columns(df_symbol)
# Join with our result dataframe
result_df = pd.merge(result_df, temp_df, on="tstamp", how="left")
result_df = result_df.reset_index(
drop=True
) # do not dropna() since irrelevant symbol would affect dataset
return result_df.dropna()
class ResearchMarketData(PtMarketData):
current_index_: int
is_execution_price_: bool
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
self.current_index_ = 0
self.is_execution_price_ = self.config_.key_exists("execution_price")
if self.is_execution_price_:
self.execution_price_column_ = self.config_.get_value("execution_price")["column"]
self.execution_price_shift_ = self.config_.get_value("execution_price")["shift"]
else:
self.execution_price_column_ = None
self.execution_price_shift_ = 0
def has_next(self) -> bool:
return self.current_index_ < len(self.market_data_df_)
def get_next(self) -> pd.Series:
result = self.market_data_df_.iloc[self.current_index_]
self.current_index_ += 1
return result
def load(self) -> None:
datafiles: List[str] = self.config_.get_value("datafiles", [])
assert len(datafiles) > 0, "No datafiles found in config"
extra_minutes: int = self.execution_price_shift_
for datafile in datafiles:
md_df = load_market_data(
datafile=datafile,
instruments=self.instruments_,
db_table_name=self.config_.get_value("market_data_loading")[
self.instruments_[0].user_data_.get("instrument_type", "?instrument_type?")
]["db_table_name"],
trading_hours=self.config_.get_value("trading_hours"),
extra_minutes=extra_minutes,
)
self.origin_mkt_data_df_ = pd.concat([self.origin_mkt_data_df_, md_df])
self.origin_mkt_data_df_ = self.origin_mkt_data_df_.sort_values(by="tstamp")
self.origin_mkt_data_df_ = self.origin_mkt_data_df_.dropna().reset_index(
drop=True
)
self.set_market_data()
self._set_execution_price_data()
def _set_execution_price_data(self) -> None:
if not self.is_execution_price_:
return
if not self.config_.key_exists("execution_price"):
self.market_data_df_[f"exec_price_{self.symbol_a_}"] = self.market_data_df_[
f"{self.stat_model_price_}_{self.symbol_a_}"
]
self.market_data_df_[f"exec_price_{self.symbol_b_}"] = self.market_data_df_[
f"{self.stat_model_price_}_{self.symbol_b_}"
]
return
execution_price_column = self.config_.get_value("execution_price")["column"]
execution_price_shift = self.config_.get_value("execution_price")["shift"]
self.market_data_df_[f"exec_price_{self.symbol_a_}"] = self.market_data_df_[
f"{execution_price_column}_{self.symbol_a_}"
].shift(-execution_price_shift)
self.market_data_df_[f"exec_price_{self.symbol_b_}"] = self.market_data_df_[
f"{execution_price_column}_{self.symbol_b_}"
].shift(-execution_price_shift)
self.market_data_df_ = self.market_data_df_.dropna().reset_index(drop=True)
def md_columns(self) -> List[str]:
# @abstractmethod
if self.is_execution_price_:
return ["tstamp", "symbol", self.stat_model_price_, self.execution_price_column_]
else:
return ["tstamp", "symbol", self.stat_model_price_]
def rename_columns(self, selected_symbol_df: pd.DataFrame) -> pd.DataFrame:
# @abstractmethod
symbol = selected_symbol_df.iloc[0]["symbol"]
new_price_column = f"{self.stat_model_price_}_{symbol}"
if self.is_execution_price_:
new_execution_price_column = f"{self.execution_price_column_}_{symbol}"
# Create temporary dataframe with timestamp and price
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
new_execution_price_column: selected_symbol_df[self.execution_price_column_],
}
)
else:
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
}
)
return temp_df
def tranform_df_target_colnames(self):
# @abstractmethod
return self.colnames() + self.orig_exec_prices_colnames()
def orig_exec_prices_colnames(self) -> List[str]:
return [
f"{self.execution_price_column_}_{self.symbol_a_}",
f"{self.execution_price_column_}_{self.symbol_b_}",
] if self.is_execution_price_ else []
class LiveMarketData(PtMarketData):
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
def md_columns(self) -> List[str]:
# @abstractmethod
return ["tstamp", "symbol", self.stat_model_price_]
def rename_columns(self, selected_symbol_df: pd.DataFrame) -> pd.DataFrame:
# @abstractmethod
symbol = selected_symbol_df.iloc[0]["symbol"]
new_price_column = f"{self.stat_model_price_}_{symbol}"
temp_df = pd.DataFrame(
{
"tstamp": selected_symbol_df["tstamp"],
new_price_column: selected_symbol_df[self.stat_model_price_],
}
)
return temp_df
def tranform_df_target_colnames(self):
# @abstractmethod
return self.colnames()
__SAV__/lib/pt_strategy/pt_model.py
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, Dict, cast
# ---
from cvttpy_tools.base.config import Config
# ---
from pairs_trading.lib.pt_strategy.prediction import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
class PairsTradingModel(ABC):
@abstractmethod
def predict(self, pair: TradingPair) -> Prediction: # type: ignore[assignment]
...
@staticmethod
def create(config: Config) -> PairsTradingModel:
import importlib
model_class_name = config.get_value("model/model_class", None)
assert model_class_name is not None
module_name, class_name = model_class_name.rsplit(".", 1)
module = importlib.import_module(module_name)
model_object = getattr(module, class_name)()
return cast(PairsTradingModel, model_object)
__SAV__/lib/pt_strategy/research_strategy.py
+305
View File
@@ -0,0 +1,305 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
# ---
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.pt_market_data import ResearchMarketData
from pairs_trading.lib.pt_strategy.pt_model import Prediction
from pairs_trading.lib.pt_strategy.trading_pair import PairState, TradingPair, ResearchTradingPair
class PtResearchStrategy:
config_: Config
trading_pair_: ResearchTradingPair
model_data_policy_: ModelDataPolicy
pt_mkt_data_: ResearchMarketData
trades_: List[pd.DataFrame]
predictions_df_: pd.DataFrame
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument]
):
from pairs_trading.lib.pt_strategy.model_data_policy import ModelDataPolicy
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
self.config_ = config
self.trades_ = []
self.trading_pair_ = ResearchTradingPair(config=config, instruments=instruments)
self.predictions_df_ = pd.DataFrame()
import copy
# modified config must be passed to PtMarketData
config_copy = copy.deepcopy(config)
config_copy.set_value("instruments", instruments)
self.pt_mkt_data_ = ResearchMarketData(config=config_copy, instruments=instruments)
self.pt_mkt_data_.load()
self.model_data_policy_ = ModelDataPolicy.create(
config_copy, mkt_data=self.pt_mkt_data_.market_data_df_, pair=self.trading_pair_
)
def outstanding_positions(self) -> List[Dict[str, Any]]:
return list(self.trading_pair_.user_data_.get("outstanding_positions", []))
def run(self) -> None:
training_minutes = self.config_.get_value("training_minutes", 120)
market_data_series: pd.Series
market_data_df = pd.DataFrame()
idx = 0
while self.pt_mkt_data_.has_next():
market_data_series = self.pt_mkt_data_.get_next()
new_row = pd.DataFrame([market_data_series])
market_data_df = pd.concat([market_data_df, new_row], ignore_index=True)
if idx >= training_minutes:
break
idx += 1
assert idx >= training_minutes, "Not enough training data"
while self.pt_mkt_data_.has_next():
market_data_series = self.pt_mkt_data_.get_next()
new_row = pd.DataFrame([market_data_series])
market_data_df = pd.concat([market_data_df, new_row], ignore_index=True)
prediction = self.trading_pair_.run(
market_data_df, self.model_data_policy_.advance(mkt_data_df=market_data_df)
)
self.predictions_df_ = pd.concat(
[self.predictions_df_, prediction.to_df()], ignore_index=True
)
assert prediction is not None
trades = self._create_trades(
prediction=prediction, last_row=market_data_df.iloc[-1]
)
if trades is not None:
self.trades_.append(trades)
trades = self._handle_outstanding_positions()
if trades is not None:
self.trades_.append(trades)
def _create_trades(
self, prediction: Prediction, last_row: pd.Series
) -> Optional[pd.DataFrame]:
pair = self.trading_pair_
trades = None
open_threshold = self.config_.get_value("model/disequilibrium/open_trshld")
close_threshold = self.config_.get_value("model/disequilibrium/close_trshld")
scaled_disequilibrium = prediction.scaled_disequilibrium_
abs_scaled_disequilibrium = abs(scaled_disequilibrium)
if pair.user_data_["state"] in [
PairState.INITIAL,
PairState.CLOSE,
PairState.CLOSE_POSITION,
PairState.CLOSE_STOP_LOSS,
PairState.CLOSE_STOP_PROFIT,
]:
if abs_scaled_disequilibrium >= open_threshold:
trades = self._create_open_trades(
pair, row=last_row, prediction=prediction
)
if trades is not None:
trades["status"] = PairState.OPEN.name
print(f"OPEN TRADES:\n{trades}")
pair.user_data_["state"] = PairState.OPEN
pair.on_open_trades(trades)
elif pair.user_data_["state"] == PairState.OPEN:
if abs_scaled_disequilibrium <= close_threshold:
trades = self._create_close_trades(
pair, row=last_row, prediction=prediction
)
if trades is not None:
trades["status"] = PairState.CLOSE.name
print(f"CLOSE TRADES:\n{trades}")
pair.user_data_["state"] = PairState.CLOSE
pair.on_close_trades(trades)
elif pair.to_stop_close_conditions(predicted_row=last_row):
trades = self._create_close_trades(pair, row=last_row)
if trades is not None:
trades["status"] = pair.user_data_["stop_close_state"].name
print(f"STOP CLOSE TRADES:\n{trades}")
pair.user_data_["state"] = pair.user_data_["stop_close_state"]
pair.on_close_trades(trades)
return trades
def _handle_outstanding_positions(self) -> Optional[pd.DataFrame]:
trades = None
pair = self.trading_pair_
# Outstanding positions
if pair.user_data_["state"] == PairState.OPEN:
print(f"{pair}: *** Position is NOT CLOSED. ***")
# outstanding positions
if self.config_.get_value("close_outstanding_positions", False):
close_position_row = pd.Series(pair.market_data_.iloc[-2])
# close_position_row["disequilibrium"] = 0.0
# close_position_row["scaled_disequilibrium"] = 0.0
# close_position_row["signed_scaled_disequilibrium"] = 0.0
trades = self._create_close_trades(
pair=pair, row=close_position_row, prediction=None
)
if trades is not None:
trades["status"] = PairState.CLOSE_POSITION.name
print(f"CLOSE_POSITION TRADES:\n{trades}")
pair.user_data_["state"] = PairState.CLOSE_POSITION
pair.on_close_trades(trades)
else:
pair.add_outstanding_position(
symbol=pair.symbol_a(),
open_side=pair.user_data_["open_side_a"],
open_px=pair.user_data_["open_px_a"],
open_tstamp=pair.user_data_["open_tstamp"],
last_mkt_data_row=pair.market_data_.iloc[-1],
)
pair.add_outstanding_position(
symbol=pair.symbol_b(),
open_side=pair.user_data_["open_side_b"],
open_px=pair.user_data_["open_px_b"],
open_tstamp=pair.user_data_["open_tstamp"],
last_mkt_data_row=pair.market_data_.iloc[-1],
)
return trades
def _trades_df(self) -> pd.DataFrame:
types = {
"time": "datetime64[ns]",
"action": "string",
"symbol": "string",
"side": "string",
"price": "float64",
"disequilibrium": "float64",
"scaled_disequilibrium": "float64",
"signed_scaled_disequilibrium": "float64",
# "pair": "object",
}
columns = list(types.keys())
return pd.DataFrame(columns=columns).astype(types)
def _create_open_trades(
self, pair: ResearchTradingPair, row: pd.Series, prediction: Prediction
) -> Optional[pd.DataFrame]:
colname_a, colname_b = pair.exec_prices_colnames()
tstamp = row["tstamp"]
diseqlbrm = prediction.disequilibrium_
scaled_disequilibrium = prediction.scaled_disequilibrium_
px_a = row[f"{colname_a}"]
px_b = row[f"{colname_b}"]
# creating the trades
df = self._trades_df()
print(f"OPEN_TRADES: {row["tstamp"]} {scaled_disequilibrium=}")
if diseqlbrm > 0:
side_a = "SELL"
side_b = "BUY"
else:
side_a = "BUY"
side_b = "SELL"
# save closing sides
pair.user_data_["open_side_a"] = side_a # used in oustanding positions
pair.user_data_["open_side_b"] = side_b
pair.user_data_["open_px_a"] = px_a
pair.user_data_["open_px_b"] = px_b
pair.user_data_["open_tstamp"] = tstamp
pair.user_data_["close_side_a"] = side_b # used for closing trades
pair.user_data_["close_side_b"] = side_a
# create opening trades
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_a(),
"side": side_a,
"action": "OPEN",
"price": px_a,
"disequilibrium": diseqlbrm,
"signed_scaled_disequilibrium": scaled_disequilibrium,
"scaled_disequilibrium": abs(scaled_disequilibrium),
# "pair": pair,
}
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_b(),
"side": side_b,
"action": "OPEN",
"price": px_b,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": abs(scaled_disequilibrium),
"signed_scaled_disequilibrium": scaled_disequilibrium,
# "pair": pair,
}
return df
def _create_close_trades(
self, pair: ResearchTradingPair, row: pd.Series, prediction: Optional[Prediction] = None
) -> Optional[pd.DataFrame]:
colname_a, colname_b = pair.exec_prices_colnames()
tstamp = row["tstamp"]
if prediction is not None:
diseqlbrm = prediction.disequilibrium_
signed_scaled_disequilibrium = prediction.scaled_disequilibrium_
scaled_disequilibrium = abs(prediction.scaled_disequilibrium_)
else:
diseqlbrm = 0.0
signed_scaled_disequilibrium = 0.0
scaled_disequilibrium = 0.0
px_a = row[f"{colname_a}"]
px_b = row[f"{colname_b}"]
# creating the trades
df = self._trades_df()
# create opening trades
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_a(),
"side": pair.user_data_["close_side_a"],
"action": "CLOSE",
"price": px_a,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": scaled_disequilibrium,
"signed_scaled_disequilibrium": signed_scaled_disequilibrium,
# "pair": pair,
}
df.loc[len(df)] = {
"time": tstamp,
"symbol": pair.symbol_b(),
"side": pair.user_data_["close_side_b"],
"action": "CLOSE",
"price": px_b,
"disequilibrium": diseqlbrm,
"scaled_disequilibrium": scaled_disequilibrium,
"signed_scaled_disequilibrium": signed_scaled_disequilibrium,
# "pair": pair,
}
del pair.user_data_["close_side_a"]
del pair.user_data_["close_side_b"]
del pair.user_data_["open_tstamp"]
del pair.user_data_["open_px_a"]
del pair.user_data_["open_px_b"]
del pair.user_data_["open_side_a"]
del pair.user_data_["open_side_b"]
return df
def day_trades(self) -> pd.DataFrame:
return pd.concat(self.trades_, ignore_index=True)
__SAV__/lib/pt_strategy/results.py
+527
View File
@@ -0,0 +1,527 @@
import os
import sqlite3
from datetime import date, datetime
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
# ---
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.trading_pair import TradingPair
# Recommended replacement adapters and converters for Python 3.12+
# From: https://docs.python.org/3/library/sqlite3.html#sqlite3-adapter-converter-recipes
def adapt_date_iso(val: date) -> str:
"""Adapt datetime.date to ISO 8601 date."""
return val.isoformat()
def adapt_datetime_iso(val: datetime) -> str:
"""Adapt datetime.datetime to timezone-naive ISO 8601 date."""
return val.isoformat()
def convert_date(val: bytes) -> date:
"""Convert ISO 8601 date to datetime.date object."""
return datetime.fromisoformat(val.decode()).date()
def convert_datetime(val: bytes) -> datetime:
"""Convert ISO 8601 datetime to datetime.datetime object."""
return datetime.fromisoformat(val.decode())
# Register the adapters and converters
sqlite3.register_adapter(date, adapt_date_iso)
sqlite3.register_adapter(datetime, adapt_datetime_iso)
sqlite3.register_converter("date", convert_date)
sqlite3.register_converter("datetime", convert_datetime)
def create_result_database(db_path: str) -> None:
"""
Create the SQLite database and required tables if they don't exist.
"""
try:
# Create directory if it doesn't exist
db_dir = os.path.dirname(db_path)
if db_dir and not os.path.exists(db_dir):
os.makedirs(db_dir, exist_ok=True)
print(f"Created directory: {db_dir}")
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Create the pt_bt_results table for completed trades
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS pt_bt_results (
date DATE,
pair TEXT,
symbol TEXT,
open_time DATETIME,
open_side TEXT,
open_price REAL,
open_quantity INTEGER,
open_disequilibrium REAL,
close_time DATETIME,
close_side TEXT,
close_price REAL,
close_quantity INTEGER,
close_disequilibrium REAL,
symbol_return REAL,
pair_return REAL,
close_condition TEXT
)
"""
)
cursor.execute("DELETE FROM pt_bt_results;")
# Create the outstanding_positions table for open positions
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS outstanding_positions (
date DATE,
pair TEXT,
symbol TEXT,
position_quantity REAL,
last_price REAL,
unrealized_return REAL,
open_price REAL,
open_side TEXT
)
"""
)
cursor.execute("DELETE FROM outstanding_positions;")
# Create the config table for storing configuration JSON for reference
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_timestamp DATETIME,
config_file_path TEXT,
config_json TEXT,
datafiles TEXT,
instruments TEXT
)
"""
)
cursor.execute("DELETE FROM config;")
conn.commit()
conn.close()
except Exception as e:
print(f"Error creating result database: {str(e)}")
raise
def store_config_in_database(
db_path: str,
config_file_path: str,
config: Config,
datafiles: List[Tuple[str, str]],
instruments: List[ExchangeInstrument],
) -> None:
"""
Store configuration information in the database for reference.
"""
import json
if db_path.upper() == "NONE":
return
try:
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
# Convert config to JSON string
config_json = json.dumps(config.data(), indent=2, default=str)
# Convert lists to comma-separated strings for storage
datafiles_str = ", ".join([f"{datafile}" for _, datafile in datafiles])
instruments_str = ", ".join(
[
inst.details_short()
for inst in instruments
]
)
# Insert configuration record
cursor.execute(
"""
INSERT INTO config (
run_timestamp, config_file_path, config_json, datafiles, instruments
) VALUES (?, ?, ?, ?, ?)
""",
(
datetime.now(),
config_file_path,
config_json,
datafiles_str,
instruments_str,
),
)
conn.commit()
conn.close()
print(f"Configuration stored in database")
except Exception as e:
print(f"Error storing configuration in database: {str(e)}")
import traceback
traceback.print_exc()
def convert_timestamp(timestamp: Any) -> Optional[datetime]:
"""Convert pandas Timestamp to Python datetime object for SQLite compatibility."""
if timestamp is None:
return None
if isinstance(timestamp, pd.Timestamp):
return timestamp.to_pydatetime()
elif isinstance(timestamp, datetime):
return timestamp
elif isinstance(timestamp, date):
return datetime.combine(timestamp, datetime.min.time())
elif isinstance(timestamp, str):
return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S")
elif isinstance(timestamp, int):
return datetime.fromtimestamp(timestamp)
else:
raise ValueError(f"Unsupported timestamp type: {type(timestamp)}")
DayT = str
TradeT = Dict[str, Any]
OutstandingPositionT = Dict[str, Any]
class PairResearchResult:
"""
Class to handle pair research results for a single pair across multiple days.
Simplified version of BacktestResult focused on single pair analysis.
"""
trades_: Dict[DayT, pd.DataFrame]
outstanding_positions_: Dict[DayT, List[OutstandingPositionT]]
symbol_roundtrip_trades_: Dict[str, List[Dict[str, Any]]]
config_: Config
def __init__(self, config: Config) -> None:
self.config_ = config
self.trades_ = {}
self.outstanding_positions_ = {}
self.total_realized_pnl = 0.0
self.symbol_roundtrip_trades_ = {}
def add_day_results(self, day: DayT, trades: pd.DataFrame, outstanding_positions: List[Dict[str, Any]]) -> None:
assert isinstance(trades, pd.DataFrame)
self.trades_[day] = trades
self.outstanding_positions_[day] = outstanding_positions
def outstanding_positions(self) -> List[OutstandingPositionT]:
"""Get all outstanding positions across all days as a flat list."""
res: List[Dict[str, Any]] = []
for day in self.outstanding_positions_.keys():
res.extend(self.outstanding_positions_[day])
return res
def calculate_returns(self) -> None:
"""Calculate and store total returns for the single pair across all days."""
self.extract_roundtrip_trades()
self.total_realized_pnl = 0.0
for day, day_trades in self.symbol_roundtrip_trades_.items():
for trade in day_trades:
self.total_realized_pnl += trade['symbol_return']
def extract_roundtrip_trades(self) -> None:
"""
Extract round-trip trades by day, grouping open/close pairs for each symbol.
Returns a dictionary with day as key and list of completed round-trip trades.
"""
def _symbol_return(trade1_side: str, trade1_px: float, trade2_side: str, trade2_px: float) -> float:
if trade1_side == "BUY" and trade2_side == "SELL":
return (trade2_px - trade1_px) / trade1_px * 100
elif trade1_side == "SELL" and trade2_side == "BUY":
return (trade1_px - trade2_px) / trade1_px * 100
else:
return 0
# Process each day separately
for day, day_trades in self.trades_.items():
# Sort trades by timestamp for the day
sorted_trades = day_trades #sorted(day_trades, key=lambda x: x["timestamp"] if x["timestamp"] else pd.Timestamp.min)
day_roundtrips = []
# Process trades in groups of 4 (open A, open B, close A, close B)
for idx in range(0, len(sorted_trades), 4):
if idx + 3 >= len(sorted_trades):
break
trade_a_1 = sorted_trades.iloc[idx] # Open A
trade_b_1 = sorted_trades.iloc[idx + 1] # Open B
trade_a_2 = sorted_trades.iloc[idx + 2] # Close A
trade_b_2 = sorted_trades.iloc[idx + 3] # Close B
# Validate trade sequence
if not (trade_a_1["action"] == "OPEN" and trade_a_2["action"] == "CLOSE"):
continue
if not (trade_b_1["action"] == "OPEN" and trade_b_2["action"] == "CLOSE"):
continue
# Calculate individual symbol returns
symbol_a_return = _symbol_return(
trade_a_1["side"], trade_a_1["price"],
trade_a_2["side"], trade_a_2["price"]
)
symbol_b_return = _symbol_return(
trade_b_1["side"], trade_b_1["price"],
trade_b_2["side"], trade_b_2["price"]
)
pair_return = symbol_a_return + symbol_b_return
# Create round-trip records for both symbols
funding_per_position = self.config_.get_value("funding_per_pair", 10000) / 2
# Symbol A round-trip
day_roundtrips.append({
"symbol": trade_a_1["symbol"],
"open_side": trade_a_1["side"],
"open_price": trade_a_1["price"],
"open_time": trade_a_1["time"],
"close_side": trade_a_2["side"],
"close_price": trade_a_2["price"],
"close_time": trade_a_2["time"],
"symbol_return": symbol_a_return,
"pair_return": pair_return,
"shares": funding_per_position / trade_a_1["price"],
"close_condition": trade_a_2.get("status", "UNKNOWN"),
"open_disequilibrium": trade_a_1.get("disequilibrium"),
"close_disequilibrium": trade_a_2.get("disequilibrium"),
})
# Symbol B round-trip
day_roundtrips.append({
"symbol": trade_b_1["symbol"],
"open_side": trade_b_1["side"],
"open_price": trade_b_1["price"],
"open_time": trade_b_1["time"],
"close_side": trade_b_2["side"],
"close_price": trade_b_2["price"],
"close_time": trade_b_2["time"],
"symbol_return": symbol_b_return,
"pair_return": pair_return,
"shares": funding_per_position / trade_b_1["price"],
"close_condition": trade_b_2.get("status", "UNKNOWN"),
"open_disequilibrium": trade_b_1.get("disequilibrium"),
"close_disequilibrium": trade_b_2.get("disequilibrium"),
})
if day_roundtrips:
self.symbol_roundtrip_trades_[day] = day_roundtrips
def print_returns_by_day(self) -> None:
"""
Print detailed return information for each day, grouped by day.
Shows individual symbol round-trips and daily totals.
"""
print("\n====== PAIR RESEARCH RETURNS BY DAY ======")
total_return_all_days = 0.0
for day, day_trades in sorted(self.symbol_roundtrip_trades_.items()):
print(f"\n--- {day} ---")
day_total_return = 0.0
pair_returns = []
# Group trades by pair (every 2 trades form a pair)
for idx in range(0, len(day_trades), 2):
if idx + 1 < len(day_trades):
trade_a = day_trades[idx]
trade_b = day_trades[idx + 1]
# Print individual symbol results
print(f" {trade_a['open_time'].time()}-{trade_a['close_time'].time()}")
print(f" {trade_a['symbol']}: {trade_a['open_side']} @ ${trade_a['open_price']:.2f} → "
f"{trade_a['close_side']} @ ${trade_a['close_price']:.2f} | "
f"Return: {trade_a['symbol_return']:+.2f}% | Shares: {trade_a['shares']:.2f}")
print(f" {trade_b['symbol']}: {trade_b['open_side']} @ ${trade_b['open_price']:.2f} → "
f"{trade_b['close_side']} @ ${trade_b['close_price']:.2f} | "
f"Return: {trade_b['symbol_return']:+.2f}% | Shares: {trade_b['shares']:.2f}")
# Show disequilibrium info if available
if trade_a.get('open_disequilibrium') is not None:
print(f" Disequilibrium: Open: {trade_a['open_disequilibrium']:.4f}, "
f"Close: {trade_a['close_disequilibrium']:.4f}")
pair_return = trade_a['pair_return']
print(f" Pair Return: {pair_return:+.2f}% | Close Condition: {trade_a['close_condition']}")
print()
pair_returns.append(pair_return)
day_total_return += pair_return
print(f" Day Total Return: {day_total_return:+.2f}% ({len(pair_returns)} pairs)")
total_return_all_days += day_total_return
print(f"\n====== TOTAL RETURN ACROSS ALL DAYS ======")
print(f"Total Return: {total_return_all_days:+.2f}%")
print(f"Total Days: {len(self.symbol_roundtrip_trades_)}")
if len(self.symbol_roundtrip_trades_) > 0:
print(f"Average Daily Return: {total_return_all_days / len(self.symbol_roundtrip_trades_):+.2f}%")
def get_return_summary(self) -> Dict[str, Any]:
"""
Get a summary of returns across all days.
Returns a dictionary with key metrics.
"""
if len(self.symbol_roundtrip_trades_) == 0:
return {
"total_return": 0.0,
"total_days": 0,
"total_pairs": 0,
"average_daily_return": 0.0,
"best_day": None,
"worst_day": None,
"daily_returns": {}
}
daily_returns = {}
total_return = 0.0
total_pairs = 0
for day, day_trades in self.symbol_roundtrip_trades_.items():
day_return = 0.0
day_pairs = len(day_trades) // 2 # Each pair has 2 symbol trades
for trade in day_trades:
day_return += trade['symbol_return']
daily_returns[day] = {
"return": day_return,
"pairs": day_pairs
}
total_return += day_return
total_pairs += day_pairs
best_day = max(daily_returns.items(), key=lambda x: x[1]["return"]) if daily_returns else None
worst_day = min(daily_returns.items(), key=lambda x: x[1]["return"]) if daily_returns else None
return {
"total_return": total_return,
"total_days": len(self.symbol_roundtrip_trades_),
"total_pairs": total_pairs,
"average_daily_return": total_return / len(self.symbol_roundtrip_trades_) if self.symbol_roundtrip_trades_ else 0.0,
"best_day": best_day,
"worst_day": worst_day,
"daily_returns": daily_returns
}
def print_grand_totals(self) -> None:
"""Print grand totals for the single pair analysis."""
summary = self.get_return_summary()
print(f"\n====== PAIR RESEARCH GRAND TOTALS ======")
print('---')
print(f"Total Return: {summary['total_return']:+.2f}%")
print('---')
print(f"Total Days Traded: {summary['total_days']}")
print(f"Total Open-Close Actions: {summary['total_pairs']}")
print(f"Total Trades: 4 * {summary['total_pairs']} = {4 * summary['total_pairs']}")
if summary['total_days'] > 0:
print(f"Average Daily Return: {summary['average_daily_return']:+.2f}%")
if summary['best_day']:
best_day, best_data = summary['best_day']
print(f"Best Day: {best_day} ({best_data['return']:+.2f}%)")
if summary['worst_day']:
worst_day, worst_data = summary['worst_day']
print(f"Worst Day: {worst_day} ({worst_data['return']:+.2f}%)")
# Update the total_realized_pnl for backward compatibility
self.total_realized_pnl = summary['total_return']
def analyze_pair_performance(self) -> None:
"""
Main method to perform comprehensive pair research analysis.
Extracts round-trip trades, calculates returns, groups by day, and prints results.
"""
print(f"\n{'='*60}")
print(f"PAIR RESEARCH PERFORMANCE ANALYSIS")
print(f"{'='*60}")
self.calculate_returns()
self.print_returns_by_day()
self.print_outstanding_positions()
self._print_additional_metrics()
self.print_grand_totals()
def _print_additional_metrics(self) -> None:
"""Print additional performance metrics."""
summary = self.get_return_summary()
if summary['total_days'] == 0:
return
print(f"\n====== ADDITIONAL METRICS ======")
# Calculate win rate
winning_days = sum(1 for day_data in summary['daily_returns'].values() if day_data['return'] > 0)
win_rate = (winning_days / summary['total_days']) * 100
print(f"Winning Days: {winning_days}/{summary['total_days']} ({win_rate:.1f}%)")
# Calculate average trade return
if summary['total_pairs'] > 0:
# Each pair has 2 symbol trades, so total symbol trades = total_pairs * 2
total_symbol_trades = summary['total_pairs'] * 2
avg_symbol_return = summary['total_return'] / total_symbol_trades
print(f"Average Symbol Return: {avg_symbol_return:+.2f}%")
avg_pair_return = summary['total_return'] / summary['total_pairs'] / 2 # Divide by 2 since we sum both symbols
print(f"Average Pair Return: {avg_pair_return:+.2f}%")
# Show daily return distribution
daily_returns_list = [data['return'] for data in summary['daily_returns'].values()]
if daily_returns_list:
print(f"Daily Return Range: {min(daily_returns_list):+.2f}% to {max(daily_returns_list):+.2f}%")
def print_outstanding_positions(self) -> None:
"""Print outstanding positions for the single pair."""
all_positions: List[OutstandingPositionT] = self.outstanding_positions()
if not all_positions:
print("\n====== NO OUTSTANDING POSITIONS ======")
return
print(f"\n====== OUTSTANDING POSITIONS ======")
print(f"{'Symbol':<10} {'Side':<4} {'Shares':<10} {'Open $':<8} {'Current $':<10} {'Value $':<12}")
print("-" * 70)
total_value = 0.0
for pos in all_positions:
current_value = pos.get("last_value", 0.0)
print(f"{pos['symbol']:<10} {pos['open_side']:<4} {pos['shares']:<10.2f} "
f"{pos['open_px']:<8.2f} {pos['last_px']:<10.2f} {current_value:<12.2f}")
total_value += current_value
print("-" * 70)
print(f"{'TOTAL VALUE':<60} ${total_value:<12.2f}")
def get_total_realized_pnl(self) -> float:
"""Get total realized PnL."""
return self.total_realized_pnl
__SAV__/lib/pt_strategy/trading_pair.py
+226
View File
@@ -0,0 +1,226 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import Any, Dict, List
import pandas as pd
# ---
from cvttpy_tools.base.base import NamedObject
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
# ---
from pairs_trading.lib.pt_strategy.model_data_policy import DataWindowParams
from pairs_trading.lib.pt_strategy.prediction import Prediction
class PairState(Enum):
INITIAL = 1
OPEN = 2
CLOSE = 3
CLOSE_POSITION = 4
CLOSE_STOP_LOSS = 5
CLOSE_STOP_PROFIT = 6
class TradingPair(NamedObject, ABC):
config_: Config
model_: Any # "PairsTradingModel"
market_data_: pd.DataFrame
user_data_: Dict[str, Any]
stat_model_price_: str
instruments_: List[ExchangeInstrument]
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument],
):
from pairs_trading.lib.pt_strategy.pt_model import PairsTradingModel
self.config_ = config
self.model_ = PairsTradingModel.create(config)
self.user_data_ = {}
self.instruments_ = instruments
self.instruments_[0].user_data_["symbol"] = instruments[0].instrument_id().split("-", 1)[1]
self.instruments_[1].user_data_["symbol"] = instruments[1].instrument_id().split("-", 1)[1]
self.stat_model_price_ = config.get_value("model/stat_model_price")
def run(self, market_data: pd.DataFrame, data_params: DataWindowParams) -> Prediction: # type: ignore[assignment]
self.market_data_ = market_data[
data_params.training_start_index_ : data_params.training_start_index_ + data_params.training_size_
]
return self.model_.predict(pair=self)
def colnames(self) -> List[str]:
return [
f"{self.stat_model_price_}_{self.symbol_a()}",
f"{self.stat_model_price_}_{self.symbol_b()}",
]
def symbol_a(self) -> str:
return self.get_instrument_a().user_data_["symbol"]
def symbol_b(self) -> str:
return self.get_instrument_b().user_data_["symbol"]
def get_instrument_a(self) -> ExchangeInstrument:
return self.instruments_[0]
def get_instrument_b(self) -> ExchangeInstrument:
return self.instruments_[1]
def __repr__(self) -> str:
return (
f"{self.__class__.__name__}:"
f" symbol_a={self.symbol_a()},"
f" symbol_b={self.symbol_b()},"
f" model={self.model_.__class__.__name__}"
)
class ResearchTradingPair(TradingPair):
def __init__(
self,
config: Config,
instruments: List[ExchangeInstrument],
):
assert len(instruments) == 2, "Trading pair must have exactly 2 instruments"
super().__init__(config=config, instruments=instruments)
self.user_data_ = {
"state": PairState.INITIAL,
}
def is_closed(self) -> bool:
return self.user_data_["state"] in [
PairState.CLOSE,
PairState.CLOSE_POSITION,
PairState.CLOSE_STOP_LOSS,
PairState.CLOSE_STOP_PROFIT,
]
def is_open(self) -> bool:
return not self.is_closed()
def exec_prices_colnames(self) -> List[str]:
return [
f"exec_price_{self.symbol_a()}",
f"exec_price_{self.symbol_b()}",
]
def to_stop_close_conditions(self, predicted_row: pd.Series) -> bool:
config = self.config_
if (
not config.key_exists("stop_close_conditions")
or config.get_value("stop_close_conditions") is None
):
return False
if "profit" in config.get_value("stop_close_conditions"):
current_return = self._current_return(predicted_row)
#
# print(f"time={predicted_row['tstamp']} current_return={current_return}")
#
if current_return >= config.get_value("stop_close_conditions")["profit"]:
print(f"STOP PROFIT: {current_return}")
self.user_data_["stop_close_state"] = PairState.CLOSE_STOP_PROFIT
return True
if "loss" in config.get_value("stop_close_conditions"):
if current_return <= config.get_value("stop_close_conditions")["loss"]:
print(f"STOP LOSS: {current_return}")
self.user_data_["stop_close_state"] = PairState.CLOSE_STOP_LOSS
return True
return False
def _current_return(self, predicted_row: pd.Series) -> float:
if "open_trades" in self.user_data_:
open_trades = self.user_data_["open_trades"]
if len(open_trades) == 0:
return 0.0
def _single_instrument_return(symbol: str) -> float:
instrument_open_trades = open_trades[open_trades["symbol"] == symbol]
instrument_open_price = instrument_open_trades["price"].iloc[0]
sign = -1 if instrument_open_trades["side"].iloc[0] == "SELL" else 1
instrument_price = predicted_row[f"{self.stat_model_price_}_{symbol}"]
instrument_return = (
sign
* (instrument_price - instrument_open_price)
/ instrument_open_price
)
return float(instrument_return) * 100.0
instrument_a_return = _single_instrument_return(self.symbol_a())
instrument_b_return = _single_instrument_return(self.symbol_b())
return instrument_a_return + instrument_b_return
return 0.0
def on_open_trades(self, trades: pd.DataFrame) -> None:
if "close_trades" in self.user_data_:
del self.user_data_["close_trades"]
self.user_data_["open_trades"] = trades
def on_close_trades(self, trades: pd.DataFrame) -> None:
del self.user_data_["open_trades"]
self.user_data_["close_trades"] = trades
def add_outstanding_position(
self,
symbol: str,
open_side: str,
open_px: float,
open_tstamp: datetime,
last_mkt_data_row: pd.Series,
) -> None:
assert symbol in [
self.symbol_a(),
self.symbol_b(),
], "Symbol must be one of the pair's symbols"
assert open_side in ["BUY", "SELL"], "Open side must be either BUY or SELL"
assert open_px > 0, "Open price must be greater than 0"
assert open_tstamp is not None, "Open timestamp must be provided"
assert last_mkt_data_row is not None, "Last market data row must be provided"
exec_prices_col_a, exec_prices_col_b = self.exec_prices_colnames()
if symbol == self.symbol_a():
last_px = last_mkt_data_row[exec_prices_col_a]
else:
last_px = last_mkt_data_row[exec_prices_col_b]
funding_per_position = self.config_.get_value("funding_per_pair") / 2
shares = funding_per_position / open_px
if open_side == "SELL":
shares = -shares
if "outstanding_positions" not in self.user_data_:
self.user_data_["outstanding_positions"] = []
self.user_data_["outstanding_positions"].append(
{
"symbol": symbol,
"open_side": open_side,
"open_px": open_px,
"shares": shares,
"open_tstamp": open_tstamp,
"last_px": last_px,
"last_tstamp": last_mkt_data_row["tstamp"],
"last_value": last_px * shares,
}
)
class LiveTradingPair(TradingPair):
def __init__(self, config: Config, instruments: List[ExchangeInstrument]):
super().__init__(config, instruments)
def to_stop_close_conditions(self, predicted_row: pd.Series) -> bool:
# TODO LiveTradingPair.to_stop_close_conditions()
return False
__SAV__/lib/tools/config.py
+17
View File
@@ -0,0 +1,17 @@
import hjson
from typing import Dict
from datetime import datetime
# ---
from cvttpy_tools.base.config import Config
def load_config(config_path: str) -> Config:
return Config(json_src=f"file://{config_path}")
def expand_filename(filename: str) -> str:
# expand %T
res = filename.replace("%T", datetime.now().strftime("%Y%m%d_%H%M%S"))
# expand %D
return res.replace("%D", datetime.now().strftime("%Y%m%d"))
__SAV__/lib/tools/data_loader.py
+150
View File
@@ -0,0 +1,150 @@
from __future__ import annotations
import sqlite3
from typing import Any, Dict, List, Tuple, cast
import pandas as pd
from cvttpy_trading.trading.instrument import ExchangeInstrument
def load_sqlite_to_dataframe(db_path:str, query:str) -> pd.DataFrame:
df: pd.DataFrame = pd.DataFrame()
import os
if not os.path.exists(db_path):
print(f"WARNING: database file {db_path} does not exist")
return df
try:
conn = sqlite3.connect(db_path)
df = pd.read_sql_query(query, conn)
return df
except sqlite3.Error as excpt:
print(f"SQLite error: {excpt}")
raise
except Exception as excpt:
print(f"Error: {excpt}")
raise Exception() from excpt
finally:
if "conn" in locals():
conn.close()
def convert_time_to_UTC(value: str, timezone: str, extra_minutes: int = 0) -> str:
from zoneinfo import ZoneInfo
from datetime import datetime, timedelta
# Parse it to naive datetime object
local_dt = datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
local_dt = local_dt + timedelta(minutes=extra_minutes)
zinfo = ZoneInfo(timezone)
result: datetime = local_dt.replace(tzinfo=zinfo).astimezone(ZoneInfo("UTC"))
return result.strftime("%Y-%m-%d %H:%M:%S")
def load_market_data(
datafile: str,
instruments: List[ExchangeInstrument],
db_table_name: str,
trading_hours: Dict = {},
extra_minutes: int = 0,
) -> pd.DataFrame:
inst_ids = ['"' + exch_inst.instrument_id() + '"' for exch_inst in instruments]
instrument_ids = list(set(inst_ids))
exchange_ids = list(
set(['"' + instrument.exchange_id() + '"' for instrument in instruments])
)
query = "select"
query += " tstamp"
query += ", tstamp_ns as time_ns"
query += f", substr(instrument_id, instr(instrument_id, '-') + 1) as symbol"
query += ", open"
query += ", high"
query += ", low"
query += ", close"
query += ", volume"
query += ", num_trades"
query += ", vwap"
query += f" from {db_table_name}"
query += f" where exchange_id in ({','.join(exchange_ids)})"
query += f" and instrument_id in ({','.join(instrument_ids)})"
df = load_sqlite_to_dataframe(db_path=datafile, query=query)
# Trading Hours
if len(df) > 0 and len(trading_hours) > 0:
date_str = df["tstamp"][0][0:10]
start_time = convert_time_to_UTC(
f"{date_str} {trading_hours['begin_session']}", trading_hours["timezone"]
)
end_time = convert_time_to_UTC(
f"{date_str} {trading_hours['end_session']}", trading_hours["timezone"], extra_minutes=extra_minutes # to get execution price
)
# Perform boolean selection
df = df[(df["tstamp"] >= start_time) & (df["tstamp"] <= end_time)]
df["tstamp"] = pd.to_datetime(df["tstamp"])
return cast(pd.DataFrame, df)
# def get_available_instruments_from_db(datafile: str, config: Dict) -> List[str]:
# """
# Auto-detect available instruments from the database by querying distinct instrument_id values.
# Returns instruments without the configured prefix.
# """
# try:
# conn = sqlite3.connect(datafile)
# # Build exclusion list with full instrument_ids
# exclude_instruments = config.get("exclude_instruments", [])
# prefix = config.get("instrument_id_pfx", "")
# exclude_instrument_ids = [f"{prefix}{inst}" for inst in exclude_instruments]
# # Query to get distinct instrument_ids
# query = f"""
# SELECT DISTINCT instrument_id
# FROM {config['db_table_name']}
# WHERE exchange_id = ?
# """
# # Add exclusion clause if there are instruments to exclude
# if exclude_instrument_ids:
# placeholders = ",".join(["?" for _ in exclude_instrument_ids])
# query += f" AND instrument_id NOT IN ({placeholders})"
# cursor = conn.execute(
# query, (config["exchange_id"],) + tuple(exclude_instrument_ids)
# )
# else:
# cursor = conn.execute(query, (config["exchange_id"],))
# instrument_ids = [row[0] for row in cursor.fetchall()]
# conn.close()
# # Remove the configured prefix to get instrument symbols
# instruments = []
# for instrument_id in instrument_ids:
# if instrument_id.startswith(prefix):
# symbol = instrument_id[len(prefix) :]
# instruments.append(symbol)
# else:
# instruments.append(instrument_id)
# return sorted(instruments)
# except Exception as e:
# print(f"Error auto-detecting instruments from {datafile}: {str(e)}")
# return []
# if __name__ == "__main__":
# df1 = load_sqlite_to_dataframe(sys.argv[1], table_name="md_1min_bars")
# print(df1)
__SAV__/lib/tools/filetools.py
+37
View File
@@ -0,0 +1,37 @@
import os
import glob
from typing import Dict, List, Tuple
# ---
from cvttpy_tools.base.config import Config
# ---
from cvttpy_trading.trading.instrument import ExchangeInstrument
DayT = str
DataFileNameT = str
def resolve_datafiles(
config: Config, date_pattern: str, instruments: List[ExchangeInstrument]
) -> List[Tuple[DayT, DataFileNameT]]:
resolved_files: List[Tuple[DayT, DataFileNameT]] = []
for exch_inst in instruments:
pattern = date_pattern
inst_type = exch_inst.user_data_.get("instrument_type", "?instrument_type?")
data_dir = config.get_value(f"market_data_loading/{inst_type}/data_directory")
if "*" in pattern or "?" in pattern:
# Handle wildcards
if not os.path.isabs(pattern):
pattern = os.path.join(data_dir, f"{pattern}.mktdata.ohlcv.db")
matched_files = glob.glob(pattern)
for matched_file in matched_files:
import re
match = re.search(r"(\d{8})\.mktdata\.ohlcv\.db$", matched_file)
assert match is not None
day = match.group(1)
resolved_files.append((day, matched_file))
else:
# Handle explicit file path
if not os.path.isabs(pattern):
pattern = os.path.join(data_dir, f"{pattern}.mktdata.ohlcv.db")
resolved_files.append((date_pattern, pattern))
return sorted(list(set(resolved_files))) # Remove duplicates and sort
__SAV__/lib/tools/viz/viz_prices.py
+79
View File
@@ -0,0 +1,79 @@
from pairs_trading.lib.pt_strategy.research_strategy import PtResearchStrategy
def visualize_prices(strategy: PtResearchStrategy, trading_date: str) -> None:
# Plot raw price data
import matplotlib.pyplot as plt
# Set plotting style
import seaborn as sns
pair = strategy.trading_pair_
SYMBOL_A = pair.symbol_a()
SYMBOL_B = pair.symbol_b()
TRD_DATE = f"{trading_date[0:4]}-{trading_date[4:6]}-{trading_date[6:8]}"
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")
plt.rcParams['figure.figsize'] = (15, 10)
# Get column names for the trading pair
colname_a, colname_b = pair.colnames()
price_data = strategy.pt_mkt_data_.market_data_df_.copy()
# Create separate subplots for better visibility
fig_price, price_axes = plt.subplots(2, 1, figsize=(18, 10))
# Plot SYMBOL_A
price_axes[0].plot(price_data['tstamp'], price_data[colname_a], alpha=0.7,
label=f'{SYMBOL_A}', linewidth=1, color='blue')
price_axes[0].set_title(f'{SYMBOL_A} Price Data ({TRD_DATE})')
price_axes[0].set_ylabel(f'{SYMBOL_A} Price')
price_axes[0].legend()
price_axes[0].grid(True)
# Plot SYMBOL_B
price_axes[1].plot(price_data['tstamp'], price_data[colname_b], alpha=0.7,
label=f'{SYMBOL_B}', linewidth=1, color='red')
price_axes[1].set_title(f'{SYMBOL_B} Price Data ({TRD_DATE})')
price_axes[1].set_ylabel(f'{SYMBOL_B} Price')
price_axes[1].set_xlabel('Time')
price_axes[1].legend()
price_axes[1].grid(True)
plt.tight_layout()
plt.show()
# Plot individual prices
fig, axes = plt.subplots(2, 1, figsize=(18, 12))
# Normalized prices for comparison
norm_a = price_data[colname_a] / price_data[colname_a].iloc[0]
norm_b = price_data[colname_b] / price_data[colname_b].iloc[0]
axes[0].plot(price_data['tstamp'], norm_a, label=f'{SYMBOL_A} (normalized)', alpha=0.8, linewidth=1)
axes[0].plot(price_data['tstamp'], norm_b, label=f'{SYMBOL_B} (normalized)', alpha=0.8, linewidth=1)
axes[0].set_title(f'Normalized Price Comparison (Base = 1.0) ({TRD_DATE})')
axes[0].set_ylabel('Normalized Price')
axes[0].legend()
axes[0].grid(True)
# Price ratio
price_ratio = price_data[colname_a] / price_data[colname_b]
axes[1].plot(price_data['tstamp'], price_ratio, label=f'{SYMBOL_A}/{SYMBOL_B} Ratio', color='green', alpha=0.8, linewidth=1)
axes[1].set_title(f'Price Ratio Px({SYMBOL_A})/Px({SYMBOL_B}) ({TRD_DATE})')
axes[1].set_ylabel('Ratio')
axes[1].set_xlabel('Time')
axes[1].legend()
axes[1].grid(True)
plt.tight_layout()
plt.show()
# Print basic statistics
print(f"\nPrice Statistics:")
print(f" {SYMBOL_A}: Mean=${price_data[colname_a].mean():.2f}, Std=${price_data[colname_a].std():.2f}")
print(f" {SYMBOL_B}: Mean=${price_data[colname_b].mean():.2f}, Std=${price_data[colname_b].std():.2f}")
print(f" Price Ratio: Mean={price_ratio.mean():.2f}, Std={price_ratio.std():.2f}")
print(f" Correlation: {price_data[colname_a].corr(price_data[colname_b]):.4f}")
__SAV__/lib/tools/viz/viz_trades.py
+502
View File
@@ -0,0 +1,502 @@
from __future__ import annotations
from pairs_trading.lib.pt_strategy.results import (PairResearchResult)
from pairs_trading.lib.pt_strategy.research_strategy import PtResearchStrategy
def visualize_trades(strategy: PtResearchStrategy, results: PairResearchResult, trading_date: str) -> None:
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import plotly.offline as pyo
from IPython.display import HTML
from plotly.subplots import make_subplots
pair = strategy.trading_pair_
trades = results.trades_[trading_date].copy()
origin_mkt_data_df = strategy.pt_mkt_data_.origin_mkt_data_df_
mkt_data_df = strategy.pt_mkt_data_.market_data_df_
TRD_DATE = f"{trading_date[0:4]}-{trading_date[4:6]}-{trading_date[6:8]}"
SYMBOL_A = pair.symbol_a()
SYMBOL_B = pair.symbol_b()
print(f"\nCreated trading pair: {pair}")
print(f"Market data shape: {pair.market_data_.shape}")
print(f"Column names: {pair.colnames()}")
# Configure plotly for offline mode
pyo.init_notebook_mode(connected=True)
# Strategy-specific interactive visualization
assert strategy.config_ is not None
print("=== SLIDING FIT INTERACTIVE VISUALIZATION ===")
print("Note: Rolling Fit strategy visualization with interactive plotly charts")
# Create consistent timeline - superset of timestamps from both dataframes
all_timestamps = sorted(set(mkt_data_df['tstamp']))
# Create a unified timeline dataframe for consistent plotting
timeline_df = pd.DataFrame({'tstamp': all_timestamps})
# Merge with predicted data to get dis-equilibrium values
timeline_df = timeline_df.merge(strategy.predictions_df_[['tstamp', 'disequilibrium', 'scaled_disequilibrium', 'signed_scaled_disequilibrium']],
on='tstamp', how='left')
# Get Symbol_A and Symbol_B market data
colname_a, colname_b = pair.colnames()
symbol_a_data = mkt_data_df[['tstamp', colname_a]].copy()
symbol_b_data = mkt_data_df[['tstamp', colname_b]].copy()
norm_a = symbol_a_data[colname_a] / symbol_a_data[colname_a].iloc[0]
norm_b = symbol_b_data[colname_b] / symbol_b_data[colname_b].iloc[0]
print(f"Using consistent timeline with {len(timeline_df)} timestamps")
print(f"Timeline range: {timeline_df['tstamp'].min()} to {timeline_df['tstamp'].max()}")
# Create subplots with price charts at bottom
fig = make_subplots(
rows=4, cols=1,
row_heights=[0.3, 0.4, 0.15, 0.15],
subplot_titles=[
f'Dis-equilibrium with Trading Thresholds ({TRD_DATE})',
f'Normalized Price Comparison with BUY/SELL Signals - {SYMBOL_A}&{SYMBOL_B} ({TRD_DATE})',
f'{SYMBOL_A} Market Data with Trading Signals ({TRD_DATE})',
f'{SYMBOL_B} Market Data with Trading Signals ({TRD_DATE})',
],
vertical_spacing=0.06,
specs=[[{"secondary_y": False}],
[{"secondary_y": False}],
[{"secondary_y": False}],
[{"secondary_y": False}]]
)
# 1. Scaled dis-equilibrium with thresholds - using consistent timeline
fig.add_trace(
go.Scatter(
x=timeline_df['tstamp'],
y=timeline_df['scaled_disequilibrium'],
name='Absolute Scaled Dis-equilibrium',
line=dict(color='green', width=2),
opacity=0.8
),
row=1, col=1
)
fig.add_trace(
go.Scatter(
x=timeline_df['tstamp'],
y=timeline_df['signed_scaled_disequilibrium'],
name='Scaled Dis-equilibrium',
line=dict(color='darkmagenta', width=2),
opacity=0.8
),
row=1, col=1
)
# Add threshold lines to first subplot
fig.add_shape(
type="line",
x0=timeline_df['tstamp'].min(),
x1=timeline_df['tstamp'].max(),
y0=strategy.config_.get_value('model/disequilibrium/open_trshld'),
y1=strategy.config_.get_value('model/disequilibrium/open_trshld'),
line=dict(color="purple", width=2, dash="dot"),
opacity=0.7,
row=1, col=1
)
fig.add_shape(
type="line",
x0=timeline_df['tstamp'].min(),
x1=timeline_df['tstamp'].max(),
y0=-strategy.config_.get_value('model/disequilibrium/open_trshld'),
y1=-strategy.config_.get_value('model/disequilibrium/open_trshld'),
line=dict(color="purple", width=2, dash="dot"),
opacity=0.7,
row=1, col=1
)
fig.add_shape(
type="line",
x0=timeline_df['tstamp'].min(),
x1=timeline_df['tstamp'].max(),
y0=strategy.config_.get_value('model/disequilibrium/close_trshld'),
y1=strategy.config_.get_value('model/disequilibrium/close_trshld'),
line=dict(color="brown", width=2, dash="dot"),
opacity=0.7,
row=1, col=1
)
fig.add_shape(
type="line",
x0=timeline_df['tstamp'].min(),
x1=timeline_df['tstamp'].max(),
y0=-strategy.config_.get_value('model/disequilibrium/close_trshld'),
y1=-strategy.config_.get_value('model/disequilibrium/close_trshld'),
line=dict(color="brown", width=2, dash="dot"),
opacity=0.7,
row=1, col=1
)
fig.add_shape(
type="line",
x0=timeline_df['tstamp'].min(),
x1=timeline_df['tstamp'].max(),
y0=0,
y1=0,
line=dict(color="black", width=1, dash="solid"),
opacity=0.5,
row=1, col=1
)
# Add normalized price lines
fig.add_trace(
go.Scatter(
x=mkt_data_df['tstamp'],
y=norm_a,
name=f'{SYMBOL_A} (Normalized)',
line=dict(color='blue', width=2),
opacity=0.8
),
row=2, col=1
)
fig.add_trace(
go.Scatter(
x=mkt_data_df['tstamp'],
y=norm_b,
name=f'{SYMBOL_B} (Normalized)',
line=dict(color='orange', width=2),
opacity=0.8,
),
row=2, col=1
)
# Add BUY and SELL signals if available
if trades is not None and len(trades) > 0:
# Define signal groups to avoid legend repetition
signal_groups = {}
# Process all trades and group by signal type (ignore OPEN/CLOSE status)
for _, trade in trades.iterrows():
symbol = trade['symbol']
side = trade['side']
# status = trade['status']
action = trade['action']
# Create signal group key (without status to combine OPEN/CLOSE)
signal_key = f"{symbol} {side} {action}"
# Find normalized price for this trade
trade_time = trade['time']
if symbol == SYMBOL_A:
closest_idx = mkt_data_df['tstamp'].searchsorted(trade_time)
if closest_idx < len(norm_a):
norm_price = norm_a.iloc[closest_idx]
else:
norm_price = norm_a.iloc[-1]
else: # SYMBOL_B
closest_idx = mkt_data_df['tstamp'].searchsorted(trade_time)
if closest_idx < len(norm_b):
norm_price = norm_b.iloc[closest_idx]
else:
norm_price = norm_b.iloc[-1]
# Initialize group if not exists
if signal_key not in signal_groups:
signal_groups[signal_key] = {
'times': [],
'prices': [],
'actual_prices': [],
'symbol': symbol,
'side': side,
# 'status': status,
'action': trade['action']
}
# Add to group
signal_groups[signal_key]['times'].append(trade_time)
signal_groups[signal_key]['prices'].append(norm_price)
signal_groups[signal_key]['actual_prices'].append(trade['price'])
# Add each signal group as a single trace
for signal_key, group_data in signal_groups.items():
symbol = group_data['symbol']
side = group_data['side']
# status = group_data['status']
# Determine marker properties (same for all OPEN/CLOSE of same side)
is_close: bool = (group_data['action'] == "CLOSE")
if 'BUY' in side:
marker_color = 'green'
marker_symbol = 'triangle-up'
marker_size = 14
else: # SELL
marker_color = 'red'
marker_symbol = 'triangle-down'
marker_size = 14
# Create hover text for each point in the group
hover_texts = []
for i, (time, norm_price, actual_price) in enumerate(zip(group_data['times'],
group_data['prices'],
group_data['actual_prices'])):
# Find the corresponding trade to get the status for hover text
trade_info = trades[(trades['time'] == time) &
(trades['symbol'] == symbol) &
(trades['side'] == side)]
if len(trade_info) > 0:
action = trade_info.iloc[0]['action']
hover_texts.append(f'<b>{signal_key} {action}</b><br>' +
f'Time: {time}<br>' +
f'Normalized Price: {norm_price:.4f}<br>' +
f'Actual Price: ${actual_price:.2f}')
else:
hover_texts.append(f'<b>{signal_key}</b><br>' +
f'Time: {time}<br>' +
f'Normalized Price: {norm_price:.4f}<br>' +
f'Actual Price: ${actual_price:.2f}')
fig.add_trace(
go.Scatter(
x=group_data['times'],
y=group_data['prices'],
mode='markers',
name=signal_key,
marker=dict(
color=marker_color,
size=marker_size,
symbol=marker_symbol,
line=dict(width=2, color='black') if is_close else None
),
showlegend=True,
hovertemplate='%{text}<extra></extra>',
text=hover_texts
),
row=2, col=1
)
# -----------------------------
fig.add_trace(
go.Scatter(
x=symbol_a_data['tstamp'],
y=symbol_a_data[colname_a],
name=f'{SYMBOL_A} Price',
line=dict(color='blue', width=2),
opacity=0.8
),
row=3, col=1
)
# Filter trades for Symbol_A
symbol_a_trades = trades[trades['symbol'] == SYMBOL_A]
print(f"\nSymbol_A trades:\n{symbol_a_trades}")
if len(symbol_a_trades) > 0:
# Separate trades by action and status for different colors
buy_open_trades = symbol_a_trades[(symbol_a_trades['side'].str.contains('BUY', na=False)) &
(symbol_a_trades['action'].str.contains('OPEN', na=False))]
buy_close_trades = symbol_a_trades[(symbol_a_trades['side'].str.contains('BUY', na=False)) &
(symbol_a_trades['action'].str.contains('CLOSE', na=False))]
sell_open_trades = symbol_a_trades[(symbol_a_trades['side'].str.contains('SELL', na=False)) &
(symbol_a_trades['action'].str.contains('OPEN', na=False))]
sell_close_trades = symbol_a_trades[(symbol_a_trades['side'].str.contains('SELL', na=False)) &
(symbol_a_trades['action'].str.contains('CLOSE', na=False))]
# Add BUY OPEN signals
if len(buy_open_trades) > 0:
fig.add_trace(
go.Scatter(
x=buy_open_trades['time'],
y=buy_open_trades['price'],
mode='markers',
name=f'{SYMBOL_A} BUY OPEN',
marker=dict(color='green', size=12, symbol='triangle-up'),
showlegend=True
),
row=3, col=1
)
# Add BUY CLOSE signals
if len(buy_close_trades) > 0:
fig.add_trace(
go.Scatter(
x=buy_close_trades['time'],
y=buy_close_trades['price'],
mode='markers',
name=f'{SYMBOL_A} BUY CLOSE',
marker=dict(color='green', size=12, symbol='triangle-up'),
line=dict(width=2, color='black'),
showlegend=True
),
row=3, col=1
)
# Add SELL OPEN signals
if len(sell_open_trades) > 0:
fig.add_trace(
go.Scatter(
x=sell_open_trades['time'],
y=sell_open_trades['price'],
mode='markers',
name=f'{SYMBOL_A} SELL OPEN',
marker=dict(color='red', size=12, symbol='triangle-down'),
showlegend=True
),
row=3, col=1
)
# Add SELL CLOSE signals
if len(sell_close_trades) > 0:
fig.add_trace(
go.Scatter(
x=sell_close_trades['time'],
y=sell_close_trades['price'],
mode='markers',
name=f'{SYMBOL_A} SELL CLOSE',
marker=dict(color='red', size=12, symbol='triangle-down'),
line=dict(width=2, color='black'),
showlegend=True
),
row=3, col=1
)
# 4. Symbol_B Market Data with Trading Signals
fig.add_trace(
go.Scatter(
x=symbol_b_data['tstamp'],
y=symbol_b_data[colname_b],
name=f'{SYMBOL_B} Price',
line=dict(color='orange', width=2),
opacity=0.8
),
row=4, col=1
)
# Add trading signals for Symbol_B if available
symbol_b_trades = trades[trades['symbol'] == SYMBOL_B]
print(f"\nSymbol_B trades:\n{symbol_b_trades}")
if len(symbol_b_trades) > 0:
# Separate trades by action and status for different colors
buy_open_trades = symbol_b_trades[(symbol_b_trades['side'].str.contains('BUY', na=False)) &
(symbol_b_trades['action'].str.startswith('OPEN', na=False))]
buy_close_trades = symbol_b_trades[(symbol_b_trades['side'].str.contains('BUY', na=False)) &
(symbol_b_trades['action'].str.startswith('CLOSE', na=False))]
sell_open_trades = symbol_b_trades[(symbol_b_trades['side'].str.contains('SELL', na=False)) &
(symbol_b_trades['action'].str.contains('OPEN', na=False))]
sell_close_trades = symbol_b_trades[(symbol_b_trades['side'].str.contains('SELL', na=False)) &
(symbol_b_trades['action'].str.contains('CLOSE', na=False))]
# Add BUY OPEN signals
if len(buy_open_trades) > 0:
fig.add_trace(
go.Scatter(
x=buy_open_trades['time'],
y=buy_open_trades['price'],
mode='markers',
name=f'{SYMBOL_B} BUY OPEN',
marker=dict(color='darkgreen', size=12, symbol='triangle-up'),
showlegend=True
),
row=4, col=1
)
# Add BUY CLOSE signals
if len(buy_close_trades) > 0:
fig.add_trace(
go.Scatter(
x=buy_close_trades['time'],
y=buy_close_trades['price'],
mode='markers',
name=f'{SYMBOL_B} BUY CLOSE',
marker=dict(color='green', size=12, symbol='triangle-up'),
line=dict(width=2, color='black'),
showlegend=True
),
row=4, col=1
)
# Add SELL OPEN signals
if len(sell_open_trades) > 0:
fig.add_trace(
go.Scatter(
x=sell_open_trades['time'],
y=sell_open_trades['price'],
mode='markers',
name=f'{SYMBOL_B} SELL OPEN',
marker=dict(color='red', size=12, symbol='triangle-down'),
showlegend=True
),
row=4, col=1
)
# Add SELL CLOSE signals
if len(sell_close_trades) > 0:
fig.add_trace(
go.Scatter(
x=sell_close_trades['time'],
y=sell_close_trades['price'],
mode='markers',
name=f'{SYMBOL_B} SELL CLOSE',
marker=dict(color='red', size=12, symbol='triangle-down'),
line=dict(width=2, color='black'),
showlegend=True
),
row=4, col=1
)
# Update layout
fig.update_layout(
height=1600,
title_text=f"Strategy Analysis - {SYMBOL_A} & {SYMBOL_B} ({TRD_DATE})",
showlegend=True,
template="plotly_white",
plot_bgcolor='lightgray',
)
# Update y-axis labels
fig.update_yaxes(title_text="Scaled Dis-equilibrium", row=1, col=1)
fig.update_yaxes(title_text=f"{SYMBOL_A} Price ($)", row=2, col=1)
fig.update_yaxes(title_text=f"{SYMBOL_B} Price ($)", row=3, col=1)
fig.update_yaxes(title_text="Normalized Price (Base = 1.0)", row=4, col=1)
# Update x-axis labels and ensure consistent time range
time_range = [timeline_df['tstamp'].min(), timeline_df['tstamp'].max()]
fig.update_xaxes(range=time_range, row=1, col=1)
fig.update_xaxes(range=time_range, row=2, col=1)
fig.update_xaxes(range=time_range, row=3, col=1)
fig.update_xaxes(title_text="Time", range=time_range, row=4, col=1)
# Display using plotly offline mode
# pyo.iplot(fig)
fig.show()
else:
print("No interactive visualization data available - strategy may not have run successfully")
print(f"\nChart shows:")
print(f"- {SYMBOL_A} and {SYMBOL_B} prices normalized to start at 1.0")
print(f"- BUY signals shown as green triangles pointing up")
print(f"- SELL signals shown as orange triangles pointing down")
print(f"- All BUY signals per symbol grouped together, all SELL signals per symbol grouped together")
print(f"- Hover over markers to see individual trade details (OPEN/CLOSE status)")
if trades is not None and len(trades) > 0:
print(f"- Total signals displayed: {len(trades)}")
print(f"- {SYMBOL_A} signals: {len(trades[trades['symbol'] == SYMBOL_A])}")
print(f"- {SYMBOL_B} signals: {len(trades[trades['symbol'] == SYMBOL_B])}")
else:
print("- No trading signals to display")