From 3b003c7811825f25dc32e7c3afbc2aa1d445f7eb Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Mon, 14 Jul 2025 00:41:46 +0000 Subject: [PATCH] progress --- configuration/crypto.cfg | 12 +- lib/pt_trading/fit_methods.py | 119 +++++++++++++----- lib/pt_trading/results.py | 6 + lib/pt_trading/trading_pair.py | 35 +++++- research/notebooks/pt_pair_backtest.ipynb | 11 +- .../equity/20250714_003409.equity_results.db | Bin 0 -> 28672 bytes sync_visualization.py | 1 + 7 files changed, 132 insertions(+), 52 deletions(-) create mode 100644 researchresults/equity/20250714_003409.equity_results.db create mode 100644 sync_visualization.py diff --git a/configuration/crypto.cfg b/configuration/crypto.cfg index eb2da76..0dd7c0d 100644 --- a/configuration/crypto.cfg +++ b/configuration/crypto.cfg @@ -7,16 +7,6 @@ "db_table_name": "md_1min_bars", "exchange_id": "BNBSPOT", "instrument_id_pfx": "PAIR-", - # "instruments": [ - # "BTC-USDT", - # "BCH-USDT", - # "ETH-USDT", - # "LTC-USDT", - # "XRP-USDT", - # "ADA-USDT", - # "SOL-USDT", - # "DOT-USDT" - # ], "trading_hours": { "begin_session": "00:00:00", "end_session": "23:59:00", @@ -29,5 +19,5 @@ "dis-equilibrium_close_trshld": 0.5, "training_minutes": 120, "funding_per_pair": 2000.0, - "fit_method_class": "pt_trading.fit_methods.StaticFit" + "fit_method_class": "pt_trading.fit_methods.SlidingFit" } \ No newline at end of file diff --git a/lib/pt_trading/fit_methods.py b/lib/pt_trading/fit_methods.py index 8cd7051..8aa088c 100644 --- a/lib/pt_trading/fit_methods.py +++ b/lib/pt_trading/fit_methods.py @@ -3,7 +3,6 @@ from enum import Enum from typing import Dict, Optional, cast import pandas as pd # type: ignore[import] - from pt_trading.results import BacktestResult from pt_trading.trading_pair import TradingPair @@ -64,6 +63,17 @@ class StaticFit(PairsTradingFitMethod): colname_a, colname_b = pair.colnames() predicted_df = pair.predicted_df_ + if predicted_df is None: + # Return empty DataFrame with correct columns and dtypes + return pd.DataFrame(columns=self.TRADES_COLUMNS).astype({ + "time": "datetime64[ns]", + "action": "string", + "symbol": "string", + "price": "float64", + "disequilibrium": "float64", + "scaled_disequilibrium": "float64", + "pair": "object" + }) open_threshold = config["dis-equilibrium_open_trshld"] close_threshold = config["dis-equilibrium_close_trshld"] @@ -96,11 +106,11 @@ class StaticFit(PairsTradingFitMethod): break open_row = predicted_df.loc[open_row_index] - open_tstamp = open_row["tstamp"] + open_px_a = predicted_df.at[open_row_index, f"{colname_a}"] + open_px_b = predicted_df.at[open_row_index, f"{colname_b}"] + open_tstamp = predicted_df.at[open_row_index, "tstamp"] open_disequilibrium = open_row["disequilibrium"] open_scaled_disequilibrium = open_row["scaled_disequilibrium"] - open_px_a = open_row[f"{colname_a}"] - open_px_b = open_row[f"{colname_b}"] abs_beta = abs(beta[1]) pred_px_b = predicted_df.loc[open_row_index][f"{colname_b}_pred"] @@ -129,9 +139,9 @@ class StaticFit(PairsTradingFitMethod): last_row_index=last_row_index, open_side_a=open_side_a, open_side_b=open_side_b, - open_px_a=open_px_a, - open_px_b=open_px_b, - open_tstamp=open_tstamp, + open_px_a=float(open_px_a), + open_px_b=float(open_px_b), + open_tstamp=pd.Timestamp(open_tstamp), ) # Return only open trades (no close trades) @@ -205,11 +215,21 @@ class StaticFit(PairsTradingFitMethod): ), ] - # Add tuples to data frame - return pd.DataFrame( + # Add tuples to data frame with explicit dtypes to avoid concatenation warnings + df = pd.DataFrame( trd_signal_tuples, - columns=self.TRADES_COLUMNS, # type: ignore + columns=self.TRADES_COLUMNS, ) + # Ensure consistent dtypes + return df.astype({ + "time": "datetime64[ns]", + "action": "string", + "symbol": "string", + "price": "float64", + "disequilibrium": "float64", + "scaled_disequilibrium": "float64", + "pair": "object" + }) def reset(self) -> None: pass @@ -232,7 +252,16 @@ class SlidingFit(PairsTradingFitMethod): print(f"***{pair}*** STARTING....") pair.user_data_["state"] = PairState.INITIAL - pair.user_data_["trades"] = pd.DataFrame(columns=self.TRADES_COLUMNS) + # Initialize trades DataFrame with proper dtypes to avoid concatenation warnings + pair.user_data_["trades"] = pd.DataFrame(columns=self.TRADES_COLUMNS).astype({ + "time": "datetime64[ns]", + "action": "string", + "symbol": "string", + "price": "float64", + "disequilibrium": "float64", + "scaled_disequilibrium": "float64", + "pair": "object" + }) pair.user_data_["is_cointegrated"] = False training_minutes = config["training_minutes"] @@ -255,17 +284,17 @@ class SlidingFit(PairsTradingFitMethod): ) # outstanding positions # last_row_index = self.curr_training_start_idx_ + training_minutes - - bt_result.handle_outstanding_position( - pair=pair, - pair_result_df=pair.predicted_df_, - last_row_index=0, - open_side_a=pair.user_data_["open_side_a"], - open_side_b=pair.user_data_["open_side_b"], - open_px_a=pair.user_data_["open_px_a"], - open_px_b=pair.user_data_["open_px_b"], - open_tstamp=pair.user_data_["open_tstamp"], - ) + if pair.predicted_df_ is not None: + bt_result.handle_outstanding_position( + pair=pair, + pair_result_df=pair.predicted_df_, + last_row_index=0, + open_side_a=pair.user_data_["open_side_a"], + open_side_b=pair.user_data_["open_side_b"], + open_px_a=pair.user_data_["open_px_a"], + open_px_b=pair.user_data_["open_px_b"], + open_tstamp=pair.user_data_["open_tstamp"], + ) break try: @@ -311,7 +340,10 @@ class SlidingFit(PairsTradingFitMethod): def _create_trading_signals( self, pair: TradingPair, config: Dict, bt_result: BacktestResult ) -> None: - assert pair.predicted_df_ is not None + if pair.predicted_df_ is None: + print(f"{pair.market_data_.iloc[0]['tstamp']} {pair}: No predicted data") + return + open_threshold = config["dis-equilibrium_open_trshld"] close_threshold = config["dis-equilibrium_close_trshld"] for curr_predicted_row_idx in range(len(pair.predicted_df_)): @@ -353,6 +385,16 @@ class SlidingFit(PairsTradingFitMethod): open_scaled_disequilibrium = open_row["scaled_disequilibrium"] open_px_a = open_row[f"{colname_a}"] open_px_b = open_row[f"{colname_b}"] + # Ensure scalars for handle_outstanding_position + if isinstance(open_px_a, pd.Series): + open_px_a = open_px_a.iloc[0] + if isinstance(open_px_b, pd.Series): + open_px_b = open_px_b.iloc[0] + if isinstance(open_tstamp, pd.Series): + open_tstamp = open_tstamp.iloc[0] + open_px_a = float(open_px_a) + open_px_b = float(open_px_b) + open_tstamp = pd.Timestamp(open_tstamp) if open_scaled_disequilibrium < open_threshold: return None @@ -402,10 +444,21 @@ class SlidingFit(PairsTradingFitMethod): pair, ), ] - return pd.DataFrame( + # Create DataFrame with explicit dtypes to avoid concatenation warnings + df = pd.DataFrame( trd_signal_tuples, - columns=self.TRADES_COLUMNS, # type: ignore + columns=self.TRADES_COLUMNS, ) + # Ensure consistent dtypes + return df.astype({ + "time": "datetime64[ns]", + "action": "string", + "symbol": "string", + "price": "float64", + "disequilibrium": "float64", + "scaled_disequilibrium": "float64", + "pair": "object" + }) def _get_close_trades( self, pair: TradingPair, row: pd.Series, close_threshold: float @@ -449,11 +502,21 @@ class SlidingFit(PairsTradingFitMethod): ), ] - # Add tuples to data frame - return pd.DataFrame( + # Add tuples to data frame with explicit dtypes to avoid concatenation warnings + df = pd.DataFrame( trd_signal_tuples, - columns=self.TRADES_COLUMNS, # type: ignore + columns=self.TRADES_COLUMNS, ) + # Ensure consistent dtypes + return df.astype({ + "time": "datetime64[ns]", + "action": "string", + "symbol": "string", + "price": "float64", + "disequilibrium": "float64", + "scaled_disequilibrium": "float64", + "pair": "object" + }) def reset(self) -> None: self.curr_training_start_idx_ = 0 diff --git a/lib/pt_trading/results.py b/lib/pt_trading/results.py index c799e0d..748bdea 100644 --- a/lib/pt_trading/results.py +++ b/lib/pt_trading/results.py @@ -41,6 +41,12 @@ def create_result_database(db_path: str) -> None: Create the SQLite database and required tables if they don't exist. """ try: + # Create directory if it doesn't exist + db_dir = os.path.dirname(db_path) + if db_dir and not os.path.exists(db_dir): + os.makedirs(db_dir, exist_ok=True) + print(f"Created directory: {db_dir}") + conn = sqlite3.connect(db_path) cursor = conn.cursor() diff --git a/lib/pt_trading/trading_pair.py b/lib/pt_trading/trading_pair.py index 6bce348..7f7e49a 100644 --- a/lib/pt_trading/trading_pair.py +++ b/lib/pt_trading/trading_pair.py @@ -179,10 +179,39 @@ class TradingPair: return result def add_trades(self, trades: pd.DataFrame) -> None: - if self.user_data_["trades"] is None: - self.user_data_["trades"] = pd.DataFrame(trades) + if self.user_data_["trades"] is None or len(self.user_data_["trades"]) == 0: + # If trades is empty or None, just assign the new trades directly + self.user_data_["trades"] = trades.copy() else: - self.user_data_["trades"] = pd.concat([self.user_data_["trades"], pd.DataFrame(trades)], ignore_index=True) + # Ensure both DataFrames have the same columns and dtypes before concatenation + existing_trades = self.user_data_["trades"] + + # If existing trades is empty, just assign the new trades + if len(existing_trades) == 0: + self.user_data_["trades"] = trades.copy() + else: + # Ensure both DataFrames have the same columns + if set(existing_trades.columns) != set(trades.columns): + # Add missing columns to trades with appropriate default values + for col in existing_trades.columns: + if col not in trades.columns: + if col == "time": + trades[col] = pd.Timestamp.now() + elif col in ["action", "symbol"]: + trades[col] = "" + elif col in ["price", "disequilibrium", "scaled_disequilibrium"]: + trades[col] = 0.0 + elif col == "pair": + trades[col] = None + else: + trades[col] = None + + # Concatenate with explicit dtypes to avoid warnings + self.user_data_["trades"] = pd.concat( + [existing_trades, trades], + ignore_index=True, + copy=False + ) def get_trades(self) -> pd.DataFrame: return self.user_data_["trades"] if "trades" in self.user_data_ else pd.DataFrame() diff --git a/research/notebooks/pt_pair_backtest.ipynb b/research/notebooks/pt_pair_backtest.ipynb index 610820a..a50eaaf 100644 --- a/research/notebooks/pt_pair_backtest.ipynb +++ b/research/notebooks/pt_pair_backtest.ipynb @@ -936,7 +936,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1075,15 +1075,6 @@ " linestyle=':', alpha=0.7)\n", " axes[1].axhline(y=0, color='black', linestyle='-', alpha=0.5, linewidth=0.5)\n", "\n", - " # if pair_trades is not None and len(pair_trades) > 0:\n", - " # # Show trading signals over time\n", - " # trade_times = pair_trades['time'].values\n", - " # trade_actions = pair_trades['action'].values\n", - " \n", - " # for i, (time, action) in enumerate(zip(trade_times, trade_actions)):\n", - " # color = 'red' if 'BUY' in action else 'blue'\n", - " # axes[1].scatter(time, i, color=color, alpha=0.8, s=50)\n", - " \n", "\n", " axes[1].set_title('Testing Period: Scaled Dis-equilibrium with Trading Thresholds')\n", " axes[1].set_ylabel('Scaled Dis-equilibrium')\n", diff --git a/researchresults/equity/20250714_003409.equity_results.db b/researchresults/equity/20250714_003409.equity_results.db new file mode 100644 index 0000000000000000000000000000000000000000..7c1c1796b93649a142a9e2ede0cab795af1215b8 GIT binary patch literal 28672 zcmeI4dvp_39>1iZh|qS}3%nDNr89w3)VpNhVDuKzW=s zv@9+@7Uigi$Lb2AySTCnsOX}_g7pE22pm?x2ZFdLASwvTYwygRY0@Us^e_K#@0=!0 zbAP|N-~0L9xpQCB>S+}eLn<3+pWDGGb&@oRR4O5qN{K{r8+ggUOE}`ehXmmd@LL-8 zS?uF&5_P>a1*9g(S4&bh4$#TJ1|QIY2p|H803v`0AOeU0B7g{l2vj8`rRU~KYZ*tK zi}W|UK>yV4Cz}JLr@rT(i6z!ji>*{?vrMQcRratQ<8ipjJm5KY+2nXxdS<4yt(iM# zJ?&|r8sU3Fw==-+lw;(|9tY)AmQS^nmX%tSRo3z;7Hh3?a%rv7Qe&$up9)SfrF5z- zFT%?g@Yos3P5K#!+pC-il-kOtlt$XYZ0!w{i?n+kOq0@9Iy1`Yc(Zwa+9P7oKrwbV z$u!YUd%ery_luZ09gKs`Oe~%9_!(coO?nt;(XC)(m$~C)=_Zr3t%ajW2N+O3kCXB= z+P$=&Vkp|8i4@V_;;y4zB8h@bdvn0yVJN0W393;= zxno6KjkDj#oc+g&63K}v$LfS z0?gow@G|x~24v?CxEO!8KjOOa?UwZ0aOFzDR`_1(diRn>GVdFzuE6nErO0`l6x(o6 zE~?H)1>7RRsdv$SlDkfOkqwQyA{3b)H0Z6eK#SPQpj** zR=TBfiRF(gM^*kgakf^YHRNkd`Iz3O(G=Cd5Pu2%~;0rs`;w7_+lv$YWMvNYp$l7 zT=nzS&blm+JQtq1&d%-NwzCf=uqSpq?U- zCy-8hoZ+dM5u}ABv)j2B(H>5(#Z7|E@Q#A15Ru1d(4#ynfMidO(D zm_{2>!}63-4Rw7r)c4iU&{soaUkwZUXozY0YKZmKu&*4kzH-F+T58PHSB>UA8fqi) zlz<15@|0y0Y$bWhdnzj@<|(IC+pKxYs@kfGJmt*VS$WD5Ye@w+{#RU{|w1YiJ~@jYij=#=YY2c4CznH z-U`rv)$k9QcnX&(8W+`)+&3^Xcnv;2-zflk@B~b5NDe z$z&CBT18~Cip6r&Xt^A{8H-MqZ5c!;(iXmc@xZEBIyFYNa9-JW%ZKl<3QmGd{L>wq zmcI1C>miv#L}}_`($uN7CX-obVl%W-r>~GV9wKtH=Ta$yHxY5k*Dn&CQD_QIkmi7W z;MgyC;Ty}oc<$j7L)vex$w~ca@4oiSkGDUf&us{*28+@Rml>TV%u#FLa&(`AIi|fj zCx1r&=`nR`*uuGYc5@PB;-?23i8m1YLox@6%GCBwQ(t5-t1+$CsDnYbZafNV=W|5P zCqI3g^wcv%T=5R=rHr#Zs!GU9$Hidv;z=V5-w~YfQ?no0oh@CBAG!Fn@%!Z|_(9CP z^y;40plYCK(8Fa$q1lz8mgC}CpkogrjbcuwAIy%Wk`IBWdESuWmGcEJL1z2b8&94( z9+w@GdAq1gEGA8@+Ei#V>NuI!nzw_{;XJ)(<36poiVBKt* z#$H2$7k+`5YybK7^7d1okMw(Anuq_pIk9+g-F-pTZK6RBml>TV%n>tlInui!PQ~Jb zFZ@_>Z!Dc!JsWiTL9WFlcqc(7{!PWiVUo<(LNe1tWoly4)SJ{gow?8eFClBjWx#VR zk^8rOhvlaoM8dIzth=6+MBwc5;_7A^Tceu=C;Z&IZ(JCfs>M66KX%l1CeL!V7D z%js?@*`90wtvc8*X$uP&J#!^XTdY%>v( znHrNOJ4MdEM7ZcpkY>#ScpJ)|tR21lm9L1niQimFIe1Oni=|mZ&lv-PK=4lZdH+|o zqF>I|{l$XbO=fhO zFh>n+TWJ0+q_FP2dyk?xmQHw2r_X|Jz@n1`@6;vp`IK=Z7jJteBvURbGe!?dU!*gt z%_bvO2;*mc1NI_vwpC{>cp^wh?7BggQhfx@E-$ujR#(9*op-{&&nx&u9$dUDdGwd# z);{~iuEe$v%#D+d1y#wSK@XQ1l_sB|+03*KElFF_{$S=x@wwL5Y{GW&_jH1Ktk?Q2HqZvvW_+K3s9h43zv#dhNs6SQ=Q{8}|VsFH~WJzQpV znlMLGA#4KQhB_4w4E0aU*%3>p8MY6#eBFdjf=v99Wc!A_<>eunNuo0Kv1n?IYRq8P z8u%WPgsFb?rzu2kXVZ^5>j)zL;jNgvb72I|E-$vKOuNoDn{xyw{K6eu-1Di2@Yj#s z7?ZPmc>8-(Gp~LstqH0UMS~tLGb&9!Lp>LCZyeul7H2HE=(rS1rCtw1a2^aya(X+kepIP0)}YvTf=1Sh=B`q_w_9C!O!$&G=tayPc0nSS%C{HHZRRlF$8aGB9* z!VI9DU?O)jzBK6l%Z8xX?7)vGJm-c4E=AHKvWa6(@UOd*at12WjPLyVNw>TLE#J$)dkNW}J zeVd)T3El}m9sf*chw3%xJ53yi5%@H{~yY zGoW%s^X?BPFNviRgNuxQ+rIiAEJ}j+>XP|i^^v+`FKi0QyhBVTX6}79#ZZKq*%cD4 z8Lr;3V>)()#GWeOKdy-w*t)JPVcnGon1U0W9<0V;;dn3n!W??^(XmVLt8;&8J29(c zSDNgV)em0r235nvg5FJLlo4x}Op}qD81b%$bBn2&wvSe>Sr$_#HwV+a;_wq0Ico^s zNszf~$R7$W?MhXKWDXUTX^Kfxr#1qbdVX$-Uq#4i-KEgqJCKkLT9QePy&8#=_kyc; z>>49j3+0{g^KW^#|A`oaA8LKSeV(!%|1`-^^VAz>f~ri>pohzhP7`LR<0c&FANV!m jDV>T1&9`Ey)WHzYSK;_y0lm6pzLvao=4;ONg3SK`%`5mL literal 0 HcmV?d00001 diff --git a/sync_visualization.py b/sync_visualization.py new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/sync_visualization.py @@ -0,0 +1 @@ + \ No newline at end of file