diff --git a/.gitignore b/.gitignore index 999e22d..ce10c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ data cvttpy # SpecStory explanation file .specstory/.what-is-this.md +results/ diff --git a/PAIRS_TRADING_BACKTEST_USAGE.md b/PAIRS_TRADING_BACKTEST_USAGE.md new file mode 100644 index 0000000..e07a6ef --- /dev/null +++ b/PAIRS_TRADING_BACKTEST_USAGE.md @@ -0,0 +1,184 @@ +# Enhanced Pairs Trading Backtest Usage Guide + +## Overview + +The enhanced `pt_backtest.py` script now supports multi-day and multi-instrument backtesting with SQLite database output. This guide explains how to use the new features. + +## New Features + +### 1. Multi-Day Data Processing +- Process multiple data files in a single run +- Support for wildcard patterns in configuration files +- CLI override for data file specification + +### 2. Dynamic Instrument Selection +- Auto-detection of instruments from database +- CLI override for instrument specification +- No need to manually update configuration files + +### 3. SQLite Database Output +- Automated storage of backtest results +- Structured data format for analysis +- Optional database output (can be disabled) + +## Command Line Arguments + +### Required Arguments +- `--config`: Path to configuration file +- `--result_db`: Path to SQLite database for results (use "NONE" to disable) + +### Optional Arguments +- `--datafiles`: Comma-separated list of data files (overrides config) +- `--instruments`: Comma-separated list of instruments (overrides auto-detection) + +## Usage Examples + +### Basic Usage (Auto-detect instruments, use config datafiles) +```bash +python src/pt_backtest.py --config configuration/crypto.cfg --result_db results.db +``` + +### Specify Instruments via CLI +```bash +python src/pt_backtest.py \ + --config configuration/crypto.cfg \ + --result_db results.db \ + --instruments "BTC-USDT,ETH-USDT,ADA-USDT" +``` + +### Override Data Files via CLI +```bash +python src/pt_backtest.py \ + --config configuration/crypto.cfg \ + --result_db results.db \ + --datafiles "20250528.mktdata.ohlcv.db,20250529.mktdata.ohlcv.db" +``` + +### Complete Override (Custom instruments and data files) +```bash +python src/pt_backtest.py \ + --config configuration/crypto.cfg \ + --result_db results.db \ + --instruments "BTC-USDT,ETH-USDT" \ + --datafiles "20250528.mktdata.ohlcv.db,20250529.mktdata.ohlcv.db" +``` + +### Disable Database Output +```bash +python src/pt_backtest.py \ + --config configuration/crypto.cfg \ + --result_db NONE +``` + +## Configuration File Updates + +### Wildcard Support in Data Files +The configuration file now supports wildcards in the `datafiles` array: + +```json +{ + "datafiles": [ + "2025*.mktdata.ohlcv.db", + "specific_file.db", + "202405*.mktdata.ohlcv.db" + ] +} +``` + +### Multiple Patterns +You can specify multiple wildcard patterns: + +```json +{ + "datafiles": [ + "202405*.mktdata.ohlcv.db", + "202406*.mktdata.ohlcv.db", + "special_data.db" + ] +} +``` + +## Database Schema + +The script creates a `pt_bt_results` table with the following schema: + +| Column | Type | Description | +|--------|------|-------------| +| date | DATE | Trading date extracted from filename | +| pair | TEXT | Trading pair name (e.g., "BTC-USDT & ETH-USDT") | +| symbol | TEXT | Individual symbol (e.g., "BTC-USDT") | +| open_time | DATETIME | Trade opening time | +| open_side | TEXT | Opening side (BUY/SELL) | +| open_price | REAL | Opening price | +| open_quantity | INTEGER | Opening quantity | +| open_disequilibrium | REAL | Disequilibrium at opening | +| close_time | DATETIME | Trade closing time | +| close_side | TEXT | Closing side (BUY/SELL) | +| close_price | REAL | Closing price | +| close_quantity | INTEGER | Closing quantity | +| close_disequilibrium | REAL | Disequilibrium at closing | +| symbol_return | REAL | Individual symbol return (%) | +| pair_return | REAL | Combined pair return (%) | + +## Auto-Detection Logic + +### Instrument Auto-Detection +When `--instruments` is not specified, the script: +1. Connects to each data file +2. Queries distinct `instrument_id` values from the configured table +3. Removes the configured prefix (`instrument_id_pfx`) +4. Uses the resulting symbols for pair generation + +### Data File Resolution +The script resolves data files in this order: +1. If `--datafiles` is specified, use those files +2. Otherwise, process each pattern in config `datafiles`: + - Expand wildcards using `glob.glob()` + - Resolve relative paths using `data_directory` + - Remove duplicates and sort + +## Output + +### Console Output +- Lists all data files to be processed +- Shows auto-detected or specified instruments +- Displays trade signals for each file +- Prints returns by day and pair +- Shows grand totals and outstanding positions + +### Database Output +- Creates database and table automatically +- Stores detailed trade information +- Includes calculated returns +- One record per symbol per trade + +## Error Handling + +The script includes comprehensive error handling: +- Invalid data files are skipped with warnings +- Database connection errors are reported +- Auto-detection failures fall back gracefully +- Processing errors are logged with stack traces + +## Performance Considerations + +- Wildcard expansion happens once at startup +- Database connections are opened/closed per operation +- Large numbers of files are processed sequentially +- Memory usage scales with the number of instruments and data points + +## Troubleshooting + +### Common Issues + +1. **No instruments found**: Check that the database contains data for the specified exchange_id +2. **No data files found**: Verify wildcard patterns and data_directory path +3. **Database errors**: Ensure write permissions for the result database path +4. **Memory issues**: Consider processing fewer files at once or reducing instrument count + +### Debug Tips + +- Use `--result_db NONE` to disable database output during testing +- Start with a small set of instruments using `--instruments` +- Test with explicit file lists using `--datafiles` before using wildcards +- Check console output for detailed processing information \ No newline at end of file diff --git a/configuration/crypto.cfg b/configuration/crypto.cfg index d12492b..b782692 100644 --- a/configuration/crypto.cfg +++ b/configuration/crypto.cfg @@ -2,21 +2,21 @@ "security_type": "CRYPTO", "data_directory": "./data/crypto", "datafiles": [ - "20250528.mktdata.ohlcv.db" + "2025*.mktdata.ohlcv.db" ], "db_table_name": "md_1min_bars", "exchange_id": "BNBSPOT", "instrument_id_pfx": "PAIR-", - "instruments": [ - "BTC-USDT", - "BCH-USDT", - "ETH-USDT", - "LTC-USDT", - "XRP-USDT", - "ADA-USDT", - "SOL-USDT", - "DOT-USDT" - ], + # "instruments": [ + # "BTC-USDT", + # "BCH-USDT", + # "ETH-USDT", + # "LTC-USDT", + # "XRP-USDT", + # "ADA-USDT", + # "SOL-USDT", + # "DOT-USDT" + # ], "trading_hours": { "begin_session": "00:00:00", "end_session": "23:59:00", diff --git a/configuration/equity.cfg b/configuration/equity.cfg index c4321b3..1c90b9d 100644 --- a/configuration/equity.cfg +++ b/configuration/equity.cfg @@ -15,10 +15,10 @@ "db_table_name": "md_1min_bars", "exchange_id": "ALPACA", "instrument_id_pfx": "STOCK-", - "instruments": [ - "COIN", - "GBTC" - ], + # "instruments": [ + # "COIN", + # "GBTC" + # ], "trading_hours": { "begin_session": "9:30:00", "end_session": "16:00:00", diff --git a/prompts/20250612.testing_code_request.txt b/prompts/20250612.testing_code_request.txt deleted file mode 100644 index 3e2eee0..0000000 --- a/prompts/20250612.testing_code_request.txt +++ /dev/null @@ -1 +0,0 @@ -Create python code that tests pairs trading strategy \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4ad8c57..0537291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,18 @@ aiohttp>=3.8.4 aiosignal>=1.3.1 -apt-clone>=0.2.1 -apturl>=0.5.2 async-timeout>=4.0.2 attrs>=21.2.0 beautifulsoup4>=4.10.0 black>=23.3.0 -blinker>=1.4 -Brlapi>=0.8.3 -ccsm>=0.9.14.1 certifi>=2020.6.20 chardet>=4.0.0 charset-normalizer>=3.1.0 click>=8.0.3 colorama>=0.4.4 -command-not-found>=0.3 -compizconfig-python>=0.9.14.1 configobj>=5.0.6 cryptography>=3.4.8 -cupshelpers>=1.0 -dbus-python>=1.2.18 -defer>=1.0.6 distro>=1.7.0 docker>=5.0.3 -docker-compose>=1.29.2 dockerpty>=0.4.1 docopt>=0.6.2 eyeD3>=0.8.10 @@ -35,20 +24,16 @@ httplib2>=0.20.2 idna>=3.3 ifaddr>=0.1.7 IMDbPY>=2021.4.18 -importlib-metadata>=4.6.4 -iotop>=0.6 jeepney>=0.7.1 jsonschema>=3.2.0 keyring>=23.5.0 launchpadlib>=1.10.16 lazr.restfulclient>=0.14.4 lazr.uri>=1.0.6 -louis>=3.20.0 lxml>=4.8.0 Mako>=1.1.3 Markdown>=3.3.6 MarkupSafe>=2.0.1 -meld>=3.20.4 more-itertools>=8.10.0 multidict>=6.0.4 mypy>=0.942 @@ -56,7 +41,6 @@ mypy-extensions>=0.4.3 netaddr>=0.8.0 netifaces>=0.11.0 oauthlib>=3.2.0 -onboard>=1.4.1 packaging>=23.1 pathspec>=0.11.1 pexpect>=4.8.0 @@ -65,27 +49,17 @@ platformdirs>=3.2.0 protobuf>=3.12.4 psutil>=5.9.0 ptyprocess>=0.7.0 -pycairo>=1.20.1 -pycups>=2.0.1 pycurl>=7.44.1 pyelftools>=0.27 Pygments>=2.11.2 -PyGObject>=3.42.1 -PyICU>=2.8.1 -PyJWT>=2.3.0 -PyNaCl>=1.5.0 pyparsing>=2.4.7 -pyparted>=3.11.7 pyrsistent>=0.18.1 -python-apt>=2.4.0+ubuntu4 -python-debian>=0.1.43+ubuntu1.1 +python-debian>=0.1.43 #+ubuntu1.1 python-dotenv>=0.19.2 python-magic>=0.4.24 -python-xapp>=2.2.2 python-xlib>=0.29 pyxdg>=0.27 -PyYAML>=5.4.1 -remarkable>=1.87 +PyYAML>=6.0 reportlab>=3.6.8 requests>=2.25.1 requests-file>=1.5.1 @@ -95,7 +69,6 @@ six>=1.16.0 soupsieve>=2.3.1 ssh-import-id>=5.11 statsmodels>=0.14.4 -systemd-python>=234 texttable>=1.6.4 tldextract>=3.1.2 tomli>=1.2.2 @@ -204,15 +177,10 @@ types-waitress>=0.1 types-Werkzeug>=1.0 types-xxhash>=2.0 typing-extensions>=3.10.0.2 -ubuntu-drivers-common>=0.0.0 -ufw>=0.36.1 Unidecode>=1.3.3 urllib3>=1.26.5 wadllib>=1.3.6 webencodings>=0.5.1 websocket-client>=1.2.3 -xdg>=5 -xkit>=0.0.0 yarl>=1.9.1 -youtube-dl>=2021.12.17 zipp>=1.0.0 diff --git a/src/pt_backtest.py b/src/pt_backtest.py index d4b05e4..a28a235 100644 --- a/src/pt_backtest.py +++ b/src/pt_backtest.py @@ -1,14 +1,18 @@ import argparse import hjson import importlib +import glob +import os +import sqlite3 +from datetime import datetime, date -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import pandas as pd from tools.data_loader import load_market_data from tools.trading_pair import TradingPair -from results import BacktestResult +from results import BacktestResult, create_result_database, store_results_in_database def load_config(config_path: str) -> Dict: @@ -17,19 +21,105 @@ def load_config(config_path: str) -> Dict: return config -def run_all_pairs( - config: Dict, datafile: str, price_column: str, bt_result: BacktestResult, strategy -) -> None: +def get_available_instruments_from_db(datafile: str, config: Dict) -> List[str]: + """ + Auto-detect available instruments from the database by querying distinct instrument_id values. + Returns instruments without the configured prefix. + """ + try: + conn = sqlite3.connect(datafile) - def _create_pairs(config: Dict) -> List[TradingPair]: + # Query to get distinct instrument_ids + query = f""" + SELECT DISTINCT instrument_id + FROM {config['db_table_name']} + WHERE exchange_id = ? + """ + + cursor = conn.execute(query, (config["exchange_id"],)) + instrument_ids = [row[0] for row in cursor.fetchall()] + conn.close() + + # Remove the configured prefix to get instrument symbols + prefix = config.get("instrument_id_pfx", "") + instruments = [] + for instrument_id in instrument_ids: + if instrument_id.startswith(prefix): + symbol = instrument_id[len(prefix) :] + instruments.append(symbol) + else: + instruments.append(instrument_id) + + return sorted(instruments) + + except Exception as e: + print(f"Error auto-detecting instruments from {datafile}: {str(e)}") + return [] + + +def resolve_datafiles(config: Dict, cli_datafiles: Optional[str] = None) -> List[str]: + """ + Resolve the list of data files to process. + CLI datafiles take priority over config datafiles. + Supports wildcards in config but not in CLI. + """ + if cli_datafiles: + # CLI override - comma-separated list, no wildcards + datafiles = [f.strip() for f in cli_datafiles.split(",")] + # Make paths absolute relative to data directory + data_dir = config.get("data_directory", "./data") + resolved_files = [] + for df in datafiles: + if not os.path.isabs(df): + df = os.path.join(data_dir, df) + resolved_files.append(df) + return resolved_files + + # Use config datafiles with wildcard support + config_datafiles = config.get("datafiles", []) + data_dir = config.get("data_directory", "./data") + resolved_files = [] + + for pattern in config_datafiles: + if "*" in pattern or "?" in pattern: + # Handle wildcards + if not os.path.isabs(pattern): + pattern = os.path.join(data_dir, pattern) + matched_files = glob.glob(pattern) + resolved_files.extend(matched_files) + else: + # Handle explicit file path + if not os.path.isabs(pattern): + pattern = os.path.join(data_dir, pattern) + resolved_files.append(pattern) + + return sorted(list(set(resolved_files))) # Remove duplicates and sort + + +def run_backtest( + config: Dict, + datafile: str, + price_column: str, + bt_result: BacktestResult, + strategy, + instruments: List[str], +) -> None: + """ + Run backtest for all pairs using the specified instruments. + """ + + def _create_pairs(config: Dict, instruments: List[str]) -> List[TradingPair]: nonlocal datafile - instruments = config["instruments"] all_indexes = range(len(instruments)) unique_index_pairs = [(i, j) for i in all_indexes for j in all_indexes if i < j] pairs = [] - market_data_df = load_market_data( - f'{config["data_directory"]}/{datafile}', config=config - ) + + # Update config to use the specified instruments + config_copy = config.copy() + config_copy["instruments"] = instruments + + market_data_df = load_market_data(datafile, config=config_copy) + for a_index, b_index in unique_index_pairs: pair = TradingPair( market_data=market_data_df, @@ -41,12 +131,13 @@ def run_all_pairs( return pairs pairs_trades = [] - for pair in _create_pairs(config): + for pair in _create_pairs(config, instruments): single_pair_trades = strategy.run_pair( pair=pair, config=config, bt_result=bt_result ) if single_pair_trades is not None and len(single_pair_trades) > 0: pairs_trades.append(single_pair_trades) + # Check if result_list has any data before concatenating if len(pairs_trades) == 0: print("No trading signals found for any pairs") @@ -57,7 +148,6 @@ def run_all_pairs( result = result.set_index("time").sort_index() bt_result.collect_single_day_results(result) - # BacktestResults.print_single_day_results() def main() -> None: @@ -65,6 +155,25 @@ def main() -> None: parser.add_argument( "--config", type=str, required=True, help="Path to the configuration file." ) + parser.add_argument( + "--datafiles", + type=str, + required=False, + help="Comma-separated list of data files (overrides config). No wildcards supported.", + ) + parser.add_argument( + "--instruments", + type=str, + required=False, + help="Comma-separated list of instrument symbols (e.g., COIN,GBTC). If not provided, auto-detects from database.", + ) + parser.add_argument( + "--result_db", + type=str, + required=True, + help="Path to SQLite database for storing results. Use 'NONE' to disable database output.", + ) + args = parser.parse_args() config: Dict = load_config(args.config) @@ -75,41 +184,85 @@ def main() -> None: module = importlib.import_module(module_name) strategy = getattr(module, class_name)() + # Resolve data files (CLI takes priority over config) + datafiles = resolve_datafiles(config, args.datafiles) + + if not datafiles: + print("No data files found to process.") + return + + print(f"Found {len(datafiles)} data files to process:") + for df in datafiles: + print(f" - {df}") + + # Create result database if needed + if args.result_db.upper() != "NONE": + create_result_database(args.result_db) + # Initialize a dictionary to store all trade results all_results: Dict[str, Dict[str, Any]] = {} bt_results = BacktestResult(config=config) # Process each data file price_column = config["price_column"] - for datafile in config["datafiles"]: - print(f"\n====== Processing {datafile} ======") - # Clear the TRADES global dictionary and reset unrealized PnL for the new file + for datafile in datafiles: + print(f"\n====== Processing {os.path.basename(datafile)} ======") + + # Clear the trades for the new file bt_results.clear_trades() + # Determine instruments to use + if args.instruments: + # Use CLI-specified instruments + instruments = [inst.strip() for inst in args.instruments.split(",")] + print(f"Using CLI-specified instruments: {instruments}") + else: + # Auto-detect instruments from database + instruments = get_available_instruments_from_db(datafile, config) + print(f"Auto-detected instruments: {instruments}") + + if not instruments: + print(f"No instruments found for {datafile}, skipping...") + continue + # Process data for this file try: - run_all_pairs( + run_backtest( config=config, datafile=datafile, price_column=price_column, bt_result=bt_results, strategy=strategy, + instruments=instruments, ) # Store results with file name as key - filename = datafile.split("/")[-1] + filename = os.path.basename(datafile) all_results[filename] = {"trades": bt_results.trades.copy()} + # Store results in database + if args.result_db.upper() != "NONE": + store_results_in_database(args.result_db, datafile, bt_results) + print(f"Successfully processed {filename}") except Exception as e: print(f"Error processing {datafile}: {str(e)}") + import traceback + + traceback.print_exc() # Calculate and print results - bt_results.calculate_returns(all_results) - bt_results.print_grand_totals() - bt_results.print_outstanding_positions() + if all_results: + bt_results.calculate_returns(all_results) + bt_results.print_grand_totals() + bt_results.print_outstanding_positions() + + if args.result_db.upper() != "NONE": + print(f"\nResults stored in database: {args.result_db}") + else: + print("No results to display.") if __name__ == "__main__": diff --git a/src/results.py b/src/results.py index 231abea..143a439 100644 --- a/src/results.py +++ b/src/results.py @@ -1,5 +1,275 @@ from typing import Any, Dict, List import pandas as pd +import sqlite3 +import os +from datetime import datetime, date + + +# Recommended replacement adapters and converters for Python 3.12+ +# From: https://docs.python.org/3/library/sqlite3.html#sqlite3-adapter-converter-recipes +def adapt_date_iso(val): + """Adapt datetime.date to ISO 8601 date.""" + return val.isoformat() + +def adapt_datetime_iso(val): + """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" + return val.isoformat() + +def convert_date(val): + """Convert ISO 8601 date to datetime.date object.""" + return datetime.fromisoformat(val.decode()).date() + +def convert_datetime(val): + """Convert ISO 8601 datetime to datetime.datetime object.""" + return datetime.fromisoformat(val.decode()) + +# Register the adapters and converters +sqlite3.register_adapter(date, adapt_date_iso) +sqlite3.register_adapter(datetime, adapt_datetime_iso) +sqlite3.register_converter("date", convert_date) +sqlite3.register_converter("datetime", convert_datetime) + + +def create_result_database(db_path: str) -> None: + """ + Create the SQLite database and pt_bt_results table if they don't exist. + """ + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Create the pt_bt_results table for completed trades + cursor.execute(''' + CREATE TABLE IF NOT EXISTS pt_bt_results ( + date DATE, + pair TEXT, + symbol TEXT, + open_time DATETIME, + open_side TEXT, + open_price REAL, + open_quantity INTEGER, + open_disequilibrium REAL, + close_time DATETIME, + close_side TEXT, + close_price REAL, + close_quantity INTEGER, + close_disequilibrium REAL, + symbol_return REAL, + pair_return REAL + ) + ''') + + # Create the outstanding_positions table for open positions + cursor.execute(''' + CREATE TABLE IF NOT EXISTS outstanding_positions ( + date DATE, + pair TEXT, + symbol TEXT, + position_quantity REAL, + last_price REAL, + unrealized_return REAL, + open_price REAL, + open_side TEXT + ) + ''') + + conn.commit() + conn.close() + + except Exception as e: + print(f"Error creating result database: {str(e)}") + raise + + +def store_results_in_database(db_path: str, datafile: str, bt_result: 'BacktestResult') -> None: + """ + Store backtest results in the SQLite database. + """ + if db_path.upper() == "NONE": + return + + def convert_timestamp(timestamp): + """Convert pandas Timestamp to Python datetime object for SQLite compatibility.""" + if timestamp is None: + return None + if hasattr(timestamp, 'to_pydatetime'): + return timestamp.to_pydatetime() + return timestamp + + try: + # Extract date from datafile name (assuming format like 20250528.mktdata.ohlcv.db) + filename = os.path.basename(datafile) + date_str = filename.split('.')[0] # Extract date part + + # Convert to proper date format + try: + date_obj = datetime.strptime(date_str, '%Y%m%d').date() + except ValueError: + # If date parsing fails, use current date + date_obj = datetime.now().date() + + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Process each trade from bt_result + trades = bt_result.get_trades() + + for pair_name, symbols in trades.items(): + # Calculate pair return for this pair + pair_return = 0.0 + pair_trades = [] + + # First pass: collect all trades and calculate returns + for symbol, symbol_trades in symbols.items(): + if len(symbol_trades) == 0: # No trades for this symbol + print(f"Warning: No trades found for symbol {symbol} in pair {pair_name}") + continue + + elif len(symbol_trades) >= 2: # Completed trades (entry + exit) + # Handle both old and new tuple formats + if len(symbol_trades[0]) == 2: # Old format: (action, price) + entry_action, entry_price = symbol_trades[0] + exit_action, exit_price = symbol_trades[1] + open_disequilibrium = 0.0 # Fallback for old format + open_scaled_disequilibrium = 0.0 + close_disequilibrium = 0.0 + close_scaled_disequilibrium = 0.0 + open_time = datetime.now() + close_time = datetime.now() + else: # New format: (action, price, disequilibrium, scaled_disequilibrium, timestamp) + entry_action, entry_price, open_disequilibrium, open_scaled_disequilibrium, open_time = symbol_trades[0] + exit_action, exit_price, close_disequilibrium, close_scaled_disequilibrium, close_time = symbol_trades[1] + + # Handle None values + open_disequilibrium = open_disequilibrium if open_disequilibrium is not None else 0.0 + open_scaled_disequilibrium = open_scaled_disequilibrium if open_scaled_disequilibrium is not None else 0.0 + close_disequilibrium = close_disequilibrium if close_disequilibrium is not None else 0.0 + close_scaled_disequilibrium = close_scaled_disequilibrium if close_scaled_disequilibrium is not None else 0.0 + + # Convert pandas Timestamps to Python datetime objects + open_time = convert_timestamp(open_time) or datetime.now() + close_time = convert_timestamp(close_time) or datetime.now() + + # Calculate actual share quantities based on funding per pair + # Split funding equally between the two positions + funding_per_position = bt_result.config["funding_per_pair"] / 2 + shares = funding_per_position / entry_price + + # Calculate symbol return + symbol_return = 0.0 + if entry_action == "BUY" and exit_action == "SELL": + symbol_return = (exit_price - entry_price) / entry_price * 100 + elif entry_action == "SELL" and exit_action == "BUY": + symbol_return = (entry_price - exit_price) / entry_price * 100 + + pair_return += symbol_return + + pair_trades.append({ + 'symbol': symbol, + 'entry_action': entry_action, + 'entry_price': entry_price, + 'exit_action': exit_action, + 'exit_price': exit_price, + 'symbol_return': symbol_return, + 'open_disequilibrium': open_disequilibrium, + 'open_scaled_disequilibrium': open_scaled_disequilibrium, + 'close_disequilibrium': close_disequilibrium, + 'close_scaled_disequilibrium': close_scaled_disequilibrium, + 'open_time': open_time, + 'close_time': close_time, + 'shares': shares, + 'is_completed': True + }) + + # Skip one-sided trades - they will be handled by outstanding_positions table + elif len(symbol_trades) == 1: + print(f"Skipping one-sided trade for {symbol} in pair {pair_name} - will be stored in outstanding_positions table") + continue + + else: + # This should not happen, but handle unexpected cases + print(f"Warning: Unexpected number of trades ({len(symbol_trades)}) for symbol {symbol} in pair {pair_name}") + continue + + # Second pass: insert completed trade records into database + for trade in pair_trades: + # Only store completed trades in pt_bt_results table + cursor.execute(''' + INSERT INTO pt_bt_results ( + date, pair, symbol, open_time, open_side, open_price, + open_quantity, open_disequilibrium, close_time, close_side, + close_price, close_quantity, close_disequilibrium, + symbol_return, pair_return + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + date_obj, + pair_name, + trade['symbol'], + trade['open_time'], + trade['entry_action'], + trade['entry_price'], + trade['shares'], + trade['open_scaled_disequilibrium'], + trade['close_time'], + trade['exit_action'], + trade['exit_price'], + trade['shares'], + trade['close_scaled_disequilibrium'], + trade['symbol_return'], + pair_return + )) + + # Store outstanding positions in separate table + outstanding_positions = bt_result.get_outstanding_positions() + for pos in outstanding_positions: + # Calculate position quantity (negative for SELL positions) + position_qty_a = pos['shares_a'] if pos['side_a'] == 'BUY' else -pos['shares_a'] + position_qty_b = pos['shares_b'] if pos['side_b'] == 'BUY' else -pos['shares_b'] + + # Calculate unrealized returns + # For symbol A: (current_price - open_price) / open_price * 100 * position_direction + unrealized_return_a = ((pos['current_px_a'] - pos['open_px_a']) / pos['open_px_a'] * 100) * (1 if pos['side_a'] == 'BUY' else -1) + unrealized_return_b = ((pos['current_px_b'] - pos['open_px_b']) / pos['open_px_b'] * 100) * (1 if pos['side_b'] == 'BUY' else -1) + + # Store outstanding position for symbol A + cursor.execute(''' + INSERT INTO outstanding_positions ( + date, pair, symbol, position_quantity, last_price, unrealized_return, open_price, open_side + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + date_obj, + pos['pair'], + pos['symbol_a'], + position_qty_a, + pos['current_px_a'], + unrealized_return_a, + pos['open_px_a'], + pos['side_a'] + )) + + # Store outstanding position for symbol B + cursor.execute(''' + INSERT INTO outstanding_positions ( + date, pair, symbol, position_quantity, last_price, unrealized_return, open_price, open_side + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + date_obj, + pos['pair'], + pos['symbol_b'], + position_qty_b, + pos['current_px_b'], + unrealized_return_b, + pos['open_px_b'], + pos['side_b'] + )) + + conn.commit() + conn.close() + + except Exception as e: + print(f"Error storing results in database: {str(e)}") + import traceback + traceback.print_exc() class BacktestResult: @@ -13,7 +283,7 @@ class BacktestResult: self.total_realized_pnl = 0.0 self.outstanding_positions: List[Dict[str, Any]] = [] - def add_trade(self, pair_nm, symbol, action, price): + def add_trade(self, pair_nm, symbol, action, price, disequilibrium=None, scaled_disequilibrium=None, timestamp=None): """Add a trade to the results tracking.""" pair_nm = str(pair_nm) @@ -21,7 +291,7 @@ class BacktestResult: self.trades[pair_nm] = {symbol: []} if symbol not in self.trades[pair_nm]: self.trades[pair_nm][symbol] = [] - self.trades[pair_nm][symbol].append((action, price)) + self.trades[pair_nm][symbol].append((action, price, disequilibrium, scaled_disequilibrium, timestamp)) def add_outstanding_position(self, position: Dict[str, Any]): """Add an outstanding position to tracking.""" @@ -59,8 +329,13 @@ class BacktestResult: action = row.action symbol = row.symbol price = row.price + disequilibrium = getattr(row, 'disequilibrium', None) + scaled_disequilibrium = getattr(row, 'scaled_disequilibrium', None) + timestamp = getattr(row, 'time', None) self.add_trade( - pair_nm=row.pair, action=action, symbol=symbol, price=price + pair_nm=row.pair, action=action, symbol=symbol, price=price, + disequilibrium=disequilibrium, scaled_disequilibrium=scaled_disequilibrium, + timestamp=timestamp ) def print_single_day_results(self): @@ -68,8 +343,10 @@ class BacktestResult: for pair, symbols in self.trades.items(): print(f"\n--- {pair} ---") for symbol, trades in symbols.items(): - for side, price in trades: - print(f"{symbol} {side} at ${price}") + for trade_data in trades: + if len(trade_data) >= 2: + side, price = trade_data[:2] + print(f"{symbol} {side} at ${price}") def print_results_summary(self, all_results): """Print summary of all processed files.""" @@ -98,9 +375,21 @@ class BacktestResult: # Calculate individual symbol returns in the pair for symbol, trades in symbols.items(): if len(trades) >= 2: # Need at least entry and exit - # Get entry and exit trades - entry_action, entry_price = trades[0] - exit_action, exit_price = trades[1] + # Get entry and exit trades - handle both old and new tuple formats + if len(trades[0]) == 2: # Old format: (action, price) + entry_action, entry_price = trades[0] + exit_action, exit_price = trades[1] + open_disequilibrium = None + open_scaled_disequilibrium = None + close_disequilibrium = None + close_scaled_disequilibrium = None + else: # New format: (action, price, disequilibrium, scaled_disequilibrium, timestamp) + entry_action, entry_price = trades[0][:2] + exit_action, exit_price = trades[1][:2] + open_disequilibrium = trades[0][2] if len(trades[0]) > 2 else None + open_scaled_disequilibrium = trades[0][3] if len(trades[0]) > 3 else None + close_disequilibrium = trades[1][2] if len(trades[1]) > 2 else None + close_scaled_disequilibrium = trades[1][3] if len(trades[1]) > 3 else None # Calculate return based on action symbol_return = 0 @@ -119,11 +408,13 @@ class BacktestResult: exit_action, exit_price, symbol_return, + open_scaled_disequilibrium, + close_scaled_disequilibrium, ) ) pair_return += symbol_return - # Print pair returns + # Print pair returns with disequilibrium information if pair_trades: print(f" {pair}:") for ( @@ -133,9 +424,15 @@ class BacktestResult: exit_action, exit_price, symbol_return, + open_scaled_disequilibrium, + close_scaled_disequilibrium, ) in pair_trades: + disequil_info = "" + if open_scaled_disequilibrium is not None and close_scaled_disequilibrium is not None: + disequil_info = f" | Open Dis-eq: {open_scaled_disequilibrium:.2f}, Close Dis-eq: {close_scaled_disequilibrium:.2f}" + print( - f" {symbol}: {entry_action} @ ${entry_price:.2f}, {exit_action} @ ${exit_price:.2f}, Return: {symbol_return:.2f}%" + f" {symbol}: {entry_action} @ ${entry_price:.2f}, {exit_action} @ ${exit_price:.2f}, Return: {symbol_return:.2f}%{disequil_info}" ) print(f" Pair Total Return: {pair_return:.2f}%") day_return += pair_return