stat_pairs_backtest/research/pt_backtest.py

import argparse
import glob
import importlib
import os
from datetime import date, datetime
from typing import Any, Dict, List, Optional

import pandas as pd

from research.research_tools import create_pairs
from tools.config import expand_filename, load_config
from pt_trading.results import (
    BacktestResult,
    create_result_database,
    store_config_in_database,
)
from pt_trading.fit_method import PairsTradingFitMethod
from pt_trading.trading_pair import TradingPair


def resolve_datafiles(
    config: Dict, date_pattern: str, instruments: List[Dict[str, str]]
) -> List[str]:
    resolved_files = []
    for inst in instruments:
        pattern = date_pattern
        inst_type = inst["instrument_type"]
        data_dir = config["market_data_loading"][inst_type]["data_directory"]
        if "*" in pattern or "?" in pattern:
            # Handle wildcards
            if not os.path.isabs(pattern):
                pattern = os.path.join(data_dir, f"{pattern}.mktdata.ohlcv.db")
            matched_files = glob.glob(pattern)
            resolved_files.extend(matched_files)
        else:
            # Handle explicit file path
            if not os.path.isabs(pattern):
                pattern = os.path.join(data_dir, f"{pattern}.mktdata.ohlcv.db")
            resolved_files.append(pattern)
    return sorted(list(set(resolved_files)))  # Remove duplicates and sort


def get_instruments(args: argparse.Namespace, config: Dict) -> List[Dict[str, str]]:

    instruments = [
        {
            "symbol": inst.split(":")[0],
            "instrument_type": inst.split(":")[1],
            "exchange_id": inst.split(":")[2],
            "instrument_id_pfx": config["market_data_loading"][inst.split(":")[1]][
                "instrument_id_pfx"
            ],
            "db_table_name": config["market_data_loading"][inst.split(":")[1]][
                "db_table_name"
            ],
        }
        for inst in args.instruments.split(",")
    ]
    return instruments


def run_backtest(
    config: Dict,
    datafile: str,
    price_column: str,
    fit_method: PairsTradingFitMethod,
    instruments: List[Dict[str, str]],
) -> BacktestResult:
    """
    Run backtest for all pairs using the specified instruments.
    """
    bt_result: BacktestResult = BacktestResult(config=config)

    pairs_trades = []
    pairs = create_pairs(
        datafile=datafile,
        fit_method=fit_method,
        price_column=price_column,
        config=config,
        instruments=instruments,
    )
    for pair in pairs:
        single_pair_trades = fit_method.run_pair(pair=pair, bt_result=bt_result)
        if single_pair_trades is not None and len(single_pair_trades) > 0:
            pairs_trades.append(single_pair_trades)
    print(f"pairs_trades:\n{pairs_trades}")
    # Check if result_list has any data before concatenating
    if len(pairs_trades) == 0:
        print("No trading signals found for any pairs")
        return bt_result

    bt_result.collect_single_day_results(pairs_trades)
    return bt_result


def main() -> None:
    parser = argparse.ArgumentParser(description="Run pairs trading backtest.")
    parser.add_argument(
        "--config", type=str, required=True, help="Path to the configuration file."
    )
    parser.add_argument(
        "--date_pattern",
        type=str,
        required=True,
        help="Date YYYYMMDD, allows * and ? wildcards",
    )
    parser.add_argument(
        "--instruments",
        type=str,
        required=True,
        help="Comma-separated list of instrument symbols (e.g., COIN:EQUITY,GBTC:CRYPTO)",
    )
    parser.add_argument(
        "--result_db",
        type=str,
        required=True,
        help="Path to SQLite database for storing results. Use 'NONE' to disable database output.",
    )

    args = parser.parse_args()

    config: Dict = load_config(args.config)

    # Dynamically instantiate fit method class
    fit_method_class_name = config.get("fit_method_class", None)
    assert fit_method_class_name is not None
    module_name, class_name = fit_method_class_name.rsplit(".", 1)
    module = importlib.import_module(module_name)
    fit_method = getattr(module, class_name)()

    # Resolve data files (CLI takes priority over config)
    instruments = get_instruments(args, config)
    datafiles = resolve_datafiles(config, args.date_pattern, instruments)

    if not datafiles:
        print("No data files found to process.")
        return

    print(f"Found {len(datafiles)} data files to process:")
    for df in datafiles:
        print(f"  - {df}")

    # Create result database if needed
    if args.result_db.upper() != "NONE":
        args.result_db = expand_filename(args.result_db)
        create_result_database(args.result_db)

    # Initialize a dictionary to store all trade results
    all_results: Dict[str, Dict[str, Any]] = {}

    # Store configuration in database for reference
    if args.result_db.upper() != "NONE":
        # Get list of all instruments for storage

        # Remove duplicates while preserving order

        store_config_in_database(
            db_path=args.result_db,
            config_file_path=args.config,
            config=config,
            fit_method_class=fit_method_class_name,
            datafiles=datafiles,
            instruments=instruments,
        )

    # Process each data file
    price_column = config["price_column"]

    for datafile in datafiles:
        print(f"\n====== Processing {os.path.basename(datafile)} ======")

        # Process data for this file
        try:
            fit_method.reset()

            bt_results = run_backtest(
                config=config,
                datafile=datafile,
                price_column=price_column,
                fit_method=fit_method,
                instruments=instruments,
            )

            # Store results with file name as key
            filename = os.path.basename(datafile)
            all_results[filename] = {
                "trades": bt_results.trades.copy(),
                "outstanding_positions": bt_results.outstanding_positions.copy(),
            }

            # Store results in database
            if args.result_db.upper() != "NONE":
                bt_results.calculate_returns(
                    {
                        filename: {
                            "trades": bt_results.trades.copy(),
                            "outstanding_positions": bt_results.outstanding_positions.copy(),
                        }
                    }
                )
                bt_results.store_results_in_database(args.result_db, datafile)

            print(f"Successfully processed {filename}")

        except Exception as err:
            print(f"Error processing {datafile}: {str(err)}")
            import traceback

            traceback.print_exc()

    # Calculate and print results using a new BacktestResult instance for aggregation
    if all_results:
        aggregate_bt_results = BacktestResult(config=config)
        aggregate_bt_results.calculate_returns(all_results)
        aggregate_bt_results.print_grand_totals()
        aggregate_bt_results.print_outstanding_positions()

        if args.result_db.upper() != "NONE":
            print(f"\nResults stored in database: {args.result_db}")
    else:
        print("No results to display.")


if __name__ == "__main__":
    main()