From 2e589f7e8cffd80b2b37272d74aec9743ab87711 Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Thu, 12 Jun 2025 23:59:36 -0400 Subject: [PATCH] fixes --- configuration/crypto.cfg | 4 +- requirements.txt | 218 ++++++++++++++++++++++++++++++++++++++ src/pt_backtest.py | 17 ++- src/tools/data_loader.py | 19 ++-- src/tools/trading_pair.py | 23 ++-- 5 files changed, 249 insertions(+), 32 deletions(-) create mode 100644 requirements.txt diff --git a/configuration/crypto.cfg b/configuration/crypto.cfg index 2a49897..d12492b 100644 --- a/configuration/crypto.cfg +++ b/configuration/crypto.cfg @@ -2,9 +2,9 @@ "security_type": "CRYPTO", "data_directory": "./data/crypto", "datafiles": [ - "20250519.mktdata.ohlcv.db" + "20250528.mktdata.ohlcv.db" ], - "db_table_name": "bnbspot_ohlcv_1min", + "db_table_name": "md_1min_bars", "exchange_id": "BNBSPOT", "instrument_id_pfx": "PAIR-", "instruments": [ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4ad8c57 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,218 @@ +aiohttp>=3.8.4 +aiosignal>=1.3.1 +apt-clone>=0.2.1 +apturl>=0.5.2 +async-timeout>=4.0.2 +attrs>=21.2.0 +beautifulsoup4>=4.10.0 +black>=23.3.0 +blinker>=1.4 +Brlapi>=0.8.3 +ccsm>=0.9.14.1 +certifi>=2020.6.20 +chardet>=4.0.0 +charset-normalizer>=3.1.0 +click>=8.0.3 +colorama>=0.4.4 +command-not-found>=0.3 +compizconfig-python>=0.9.14.1 +configobj>=5.0.6 +cryptography>=3.4.8 +cupshelpers>=1.0 +dbus-python>=1.2.18 +defer>=1.0.6 +distro>=1.7.0 +docker>=5.0.3 +docker-compose>=1.29.2 +dockerpty>=0.4.1 +docopt>=0.6.2 +eyeD3>=0.8.10 +filelock>=3.6.0 +frozenlist>=1.3.3 +grpcio>=1.30.2 +html5lib>=1.1 +httplib2>=0.20.2 +idna>=3.3 +ifaddr>=0.1.7 +IMDbPY>=2021.4.18 +importlib-metadata>=4.6.4 +iotop>=0.6 +jeepney>=0.7.1 +jsonschema>=3.2.0 +keyring>=23.5.0 +launchpadlib>=1.10.16 +lazr.restfulclient>=0.14.4 +lazr.uri>=1.0.6 +louis>=3.20.0 +lxml>=4.8.0 +Mako>=1.1.3 +Markdown>=3.3.6 +MarkupSafe>=2.0.1 +meld>=3.20.4 +more-itertools>=8.10.0 +multidict>=6.0.4 +mypy>=0.942 +mypy-extensions>=0.4.3 +netaddr>=0.8.0 +netifaces>=0.11.0 +oauthlib>=3.2.0 +onboard>=1.4.1 +packaging>=23.1 +pathspec>=0.11.1 +pexpect>=4.8.0 +Pillow>=9.0.1 +platformdirs>=3.2.0 +protobuf>=3.12.4 +psutil>=5.9.0 +ptyprocess>=0.7.0 +pycairo>=1.20.1 +pycups>=2.0.1 +pycurl>=7.44.1 +pyelftools>=0.27 +Pygments>=2.11.2 +PyGObject>=3.42.1 +PyICU>=2.8.1 +PyJWT>=2.3.0 +PyNaCl>=1.5.0 +pyparsing>=2.4.7 +pyparted>=3.11.7 +pyrsistent>=0.18.1 +python-apt>=2.4.0+ubuntu4 +python-debian>=0.1.43+ubuntu1.1 +python-dotenv>=0.19.2 +python-magic>=0.4.24 +python-xapp>=2.2.2 +python-xlib>=0.29 +pyxdg>=0.27 +PyYAML>=5.4.1 +remarkable>=1.87 +reportlab>=3.6.8 +requests>=2.25.1 +requests-file>=1.5.1 +SecretStorage>=3.3.1 +setproctitle>=1.2.2 +six>=1.16.0 +soupsieve>=2.3.1 +ssh-import-id>=5.11 +statsmodels>=0.14.4 +systemd-python>=234 +texttable>=1.6.4 +tldextract>=3.1.2 +tomli>=1.2.2 +typed-ast>=1.4.3 +types-aiofiles>=0.1 +types-annoy>=1.17 +types-appdirs>=1.4 +types-atomicwrites>=1.4 +types-aws-xray-sdk>=2.8 +types-babel>=2.9 +types-backports-abc>=0.5 +types-backports.ssl-match-hostname>=3.7 +types-beautifulsoup4>=4.10 +types-bleach>=4.1 +types-boto>=2.49 +types-braintree>=4.11 +types-cachetools>=4.2 +types-caldav>=0.8 +types-certifi>=2020.4 +types-characteristic>=14.3 +types-chardet>=4.0 +types-click>=7.1 +types-click-spinner>=0.1 +types-colorama>=0.4 +types-commonmark>=0.9 +types-contextvars>=0.1 +types-croniter>=1.0 +types-cryptography>=3.3 +types-dataclasses>=0.1 +types-dateparser>=1.0 +types-DateTimeRange>=0.1 +types-decorator>=0.1 +types-Deprecated>=1.2 +types-docopt>=0.6 +types-docutils>=0.17 +types-editdistance>=0.5 +types-emoji>=1.2 +types-entrypoints>=0.3 +types-enum34>=1.1 +types-filelock>=3.2 +types-first>=2.0 +types-Flask>=1.1 +types-freezegun>=1.1 +types-frozendict>=0.1 +types-futures>=3.3 +types-html5lib>=1.1 +types-httplib2>=0.19 +types-humanfriendly>=9.2 +types-ipaddress>=1.0 +types-itsdangerous>=1.1 +types-JACK-Client>=0.1 +types-Jinja2>=2.11 +types-jmespath>=0.10 +types-jsonschema>=3.2 +types-Markdown>=3.3 +types-MarkupSafe>=1.1 +types-mock>=4.0 +types-mypy-extensions>=0.4 +types-mysqlclient>=2.0 +types-oauthlib>=3.1 +types-orjson>=3.6 +types-paramiko>=2.7 +types-Pillow>=8.3 +types-polib>=1.1 +types-prettytable>=2.1 +types-protobuf>=3.17 +types-psutil>=5.8 +types-psycopg2>=2.9 +types-pyaudio>=0.2 +types-pycurl>=0.1 +types-pyfarmhash>=0.2 +types-Pygments>=2.9 +types-PyMySQL>=1.0 +types-pyOpenSSL>=20.0 +types-pyRFC3339>=0.1 +types-pysftp>=0.2 +types-pytest-lazy-fixture>=0.6 +types-python-dateutil>=2.8 +types-python-gflags>=3.1 +types-python-nmap>=0.6 +types-python-slugify>=5.0 +types-pytz>=2021.1 +types-pyvmomi>=7.0 +types-PyYAML>=5.4 +types-redis>=3.5 +types-requests>=2.25 +types-retry>=0.9 +types-selenium>=3.141 +types-Send2Trash>=1.8 +types-setuptools>=57.4 +types-simplejson>=3.17 +types-singledispatch>=3.7 +types-six>=1.16 +types-slumber>=0.7 +types-stripe>=2.59 +types-tabulate>=0.8 +types-termcolor>=1.1 +types-toml>=0.10 +types-toposort>=1.6 +types-ttkthemes>=3.2 +types-typed-ast>=1.4 +types-tzlocal>=0.1 +types-ujson>=0.1 +types-vobject>=0.9 +types-waitress>=0.1 +types-Werkzeug>=1.0 +types-xxhash>=2.0 +typing-extensions>=3.10.0.2 +ubuntu-drivers-common>=0.0.0 +ufw>=0.36.1 +Unidecode>=1.3.3 +urllib3>=1.26.5 +wadllib>=1.3.6 +webencodings>=0.5.1 +websocket-client>=1.2.3 +xdg>=5 +xkit>=0.0.0 +yarl>=1.9.1 +youtube-dl>=2021.12.17 +zipp>=1.0.0 diff --git a/src/pt_backtest.py b/src/pt_backtest.py index 0fadcc6..d4b05e4 100644 --- a/src/pt_backtest.py +++ b/src/pt_backtest.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List import pandas as pd -from strategies import SlidingFitStrategy, StaticFitStrategy from tools.data_loader import load_market_data from tools.trading_pair import TradingPair from results import BacktestResult @@ -68,21 +67,21 @@ def main() -> None: ) args = parser.parse_args() - CONFIG = load_config(args.config) + config: Dict = load_config(args.config) # Dynamically instantiate strategy class - strategy_class_name = CONFIG.get("strategy_class", "strategies.StaticFitStrategy") + strategy_class_name = config.get("strategy_class", "strategies.StaticFitStrategy") module_name, class_name = strategy_class_name.rsplit(".", 1) module = importlib.import_module(module_name) - STRATEGY = getattr(module, class_name)() + strategy = getattr(module, class_name)() # Initialize a dictionary to store all trade results all_results: Dict[str, Dict[str, Any]] = {} - bt_results = BacktestResult(config=CONFIG) + bt_results = BacktestResult(config=config) # Process each data file - price_column = CONFIG["price_column"] - for datafile in CONFIG["datafiles"]: + price_column = config["price_column"] + for datafile in config["datafiles"]: print(f"\n====== Processing {datafile} ======") # Clear the TRADES global dictionary and reset unrealized PnL for the new file @@ -91,11 +90,11 @@ def main() -> None: # Process data for this file try: run_all_pairs( - config=CONFIG, + config=config, datafile=datafile, price_column=price_column, bt_result=bt_results, - strategy=STRATEGY, + strategy=strategy, ) # Store results with file name as key diff --git a/src/tools/data_loader.py b/src/tools/data_loader.py index c9609e1..b6f7fcb 100644 --- a/src/tools/data_loader.py +++ b/src/tools/data_loader.py @@ -1,9 +1,7 @@ -import sys import sqlite3 -from typing import Dict, Tuple +from typing import Dict import pandas as pd -from tools.trading_pair import TradingPair def load_sqlite_to_dataframe(db_path, query): @@ -15,15 +13,15 @@ def load_sqlite_to_dataframe(db_path, query): except sqlite3.Error as excpt: print(f"SQLite error: {excpt}") raise - except Exception as e: + except Exception as excpt: print(f"Error: {excpt}") - raise + raise Exception() from excpt finally: if "conn" in locals(): conn.close() -def convert_time_to_UTC(value: str, timezone: str): +def convert_time_to_UTC(value: str, timezone: str) -> str: from zoneinfo import ZoneInfo from datetime import datetime @@ -32,12 +30,9 @@ def convert_time_to_UTC(value: str, timezone: str): local_dt = datetime.strptime(value, "%Y-%m-%d %H:%M:%S") zinfo = ZoneInfo(timezone) - result = local_dt.replace(tzinfo=zinfo) + result: datetime = local_dt.replace(tzinfo=zinfo).astimezone(ZoneInfo("UTC")) - result = result.astimezone(ZoneInfo("UTC")) - result = result.strftime("%Y-%m-%d %H:%M:%S") - - return result + return result.strftime("%Y-%m-%d %H:%M:%S") def load_market_data(datafile: str, config: Dict) -> pd.DataFrame: @@ -52,7 +47,7 @@ def load_market_data(datafile: str, config: Dict) -> pd.DataFrame: query = "select" if security_type == "CRYPTO": - query += " strftime('%Y-%m-%d %H:%M:%S', tstamp/1000000000, 'unixepoch') as tstamp" + query += " strftime('%Y-%m-%d %H:%M:%S', tstamp_ns/1000000000, 'unixepoch') as tstamp" query += ", tstamp as time_ns" else: query += " tstamp" diff --git a/src/tools/trading_pair.py b/src/tools/trading_pair.py index 7c7d6e9..b62cb5d 100644 --- a/src/tools/trading_pair.py +++ b/src/tools/trading_pair.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional -import pandas as pd -from statsmodels.tsa.vector_ar.vecm import VECM +import pandas as pd #type:ignore +from statsmodels.tsa.vector_ar.vecm import VECM #type:ignore class TradingPair: market_data_: pd.DataFrame @@ -16,7 +16,7 @@ class TradingPair: testing_df_: Optional[pd.DataFrame] vecm_fit_: Optional[VECM] - + user_data_: Dict[str, Any] def __init__(self, market_data: pd.DataFrame, symbol_a: str, symbol_b: str, price_column: str): @@ -24,13 +24,13 @@ class TradingPair: self.symbol_b_ = symbol_b self.price_column_ = price_column self.market_data_ = self._transform_dataframe(market_data)[["tstamp"] + self.colnames()] - + self.training_mu_ = None self.training_std_ = None self.training_df_ = None self.testing_df_ = None self.vecm_fit_ = None - + self.user_data_ = {} def _transform_dataframe(self, df: pd.DataFrame): @@ -41,7 +41,9 @@ class TradingPair: result_df: pd.DataFrame = pd.DataFrame(df_selected["tstamp"]).drop_duplicates().reset_index(drop=True) # For each unique symbol, add a corresponding close price column - for symbol in df_selected["symbol"].unique(): + + symbols = df_selected["symbol"].unique() + for symbol in symbols: # Filter rows for this symbol df_symbol = df_selected[df_selected["symbol"] == symbol].reset_index(drop=True) @@ -60,7 +62,7 @@ class TradingPair: return result_df def get_datasets(self, training_minutes: int, training_start_index: int = 0, testing_size: Optional[int] = None) -> None: - + testing_start_index = training_start_index + training_minutes self.training_df_ = self.market_data_.iloc[training_start_index:testing_start_index, :].copy() self.training_df_ = self.training_df_.dropna().reset_index(drop=True) @@ -108,6 +110,7 @@ class TradingPair: # print('*' * 80 + '\n' + f"**************** {self} IS COINTEGRATED ****************\n" + '*' * 80) self.fit_VECM() + assert self.training_df_ is not None and self.vecm_fit_ is not None diseq_series = self.training_df_[self.colnames()] @ self.vecm_fit_.beta self.training_mu_ = diseq_series.mean().iloc[0] self.training_std_ = diseq_series.std().iloc[0] @@ -121,10 +124,12 @@ class TradingPair: return True def predict(self) -> None: + assert self.testing_df_ is not None + assert self.vecm_fit_ is not None predicted_prices = self.vecm_fit_.predict(steps=len(self.testing_df_)) # Convert prediction to a DataFrame for readability - # predicted_df = + # predicted_df = self.predicted_df_ = pd.merge( self.testing_df_.reset_index(drop=True), @@ -144,7 +149,7 @@ class TradingPair: self.predicted_df_ = self.predicted_df_.reset_index() return self.predicted_df_ - + def __repr__(self) ->str: return f"{self.symbol_a_} & {self.symbol_b_}"