research/stat_pairs_backtest
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Oleg Sheynin
2026-07-30 02:39:21 +00:00
parent f49a10f54e
commit 4ddf4017bd
41 changed files with 1570 additions and 13025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/spbt_day.py
+131 -22
View File
@@ -15,8 +15,18 @@ import pandas as pd
SELECTOR_PAIRS_COLUMNS = ("pair_name", "mr_score")
SELECTOR_PAIR_INSTRUMENT_COLUMNS = ("pair_name", "instrument_a", "instrument_b")
TRADING_INSTRUCTIONS_COLUMNS = ("time_ns", "tstamp", "data")
OHLCV_1MIN_COLUMNS = ("tstamp", "tstamp_ns", "exch_acct", "instrument_id", "close")
LEGACY_TRADING_INSTRUCTIONS_COLUMNS = ("time_ns", "tstamp", "data")
SP_QUANT_TRADING_INSTRUCTIONS_COLUMNS = (
"tstamp_ns",
"tstamp",
"action",
"quote_asset",
"assets",
"scaled_disequilibrium",
"beta",
)
MARKET_COLUMNS = ("tstamp", "tstamp_ns", "exch_acct", "instrument_id", "close")
OHLCV_1MIN_COLUMNS = MARKET_COLUMNS
INITIAL_THEO_CAPITAL_USD = 10_000.0
SQLITE_EXTENSIONS = {".db", ".sqlite", ".sqlite3"}
PAIR_NAME_DISPLAY_SUFFIX = ":USD"
@@ -32,6 +42,8 @@ SELECTED_PAIR_EXECUTION_DISPLAY_COLUMNS = [
"action",
"side",
"strength",
"scaled_disequilibrium",
"beta",
"size",
"price",
"usd_value",
@@ -379,24 +391,63 @@ def load_selector_pair_rankings(conn: sqlite3.Connection) -> pd.DataFrame:
return rank_selector_pairs(selector_pairs)
def table_column_names(conn: sqlite3.Connection, table_name: str) -> set[str]:
"""Return SQLite column names for an existing table, or an empty set."""
return {
row[0]
for row in conn.execute(
"SELECT name FROM pragma_table_info(?)",
(table_name,),
)
}
def validate_trading_instructions_table(conn: sqlite3.Connection) -> None:
"""Raise an actionable error if trading_instructions lacks required columns."""
table_info = conn.execute("PRAGMA table_info(trading_instructions)").fetchall()
if not table_info:
existing_columns = table_column_names(conn, "trading_instructions")
if not existing_columns:
raise ValueError("SQLite database is missing required table: trading_instructions")
existing_columns = {row[1] for row in table_info}
missing_columns = set(TRADING_INSTRUCTIONS_COLUMNS) - existing_columns
if missing_columns:
missing = ", ".join(sorted(missing_columns))
raise ValueError(
f"trading_instructions is missing required column(s): {missing}"
)
required_schemas = (
set(SP_QUANT_TRADING_INSTRUCTIONS_COLUMNS),
set(LEGACY_TRADING_INSTRUCTIONS_COLUMNS),
)
if any(required_schema <= existing_columns for required_schema in required_schemas):
return
missing_by_schema = [
", ".join(sorted(required_schema - existing_columns))
for required_schema in required_schemas
]
raise ValueError(
"trading_instructions does not match a supported schema; missing either "
f"SP Quant column(s) [{missing_by_schema[0]}] or legacy column(s) "
f"[{missing_by_schema[1]}]"
)
def load_trading_instructions(conn: sqlite3.Connection) -> pd.DataFrame:
"""Load the full trading_instructions table ordered by timestamp."""
"""Load trading_instructions ordered by timestamp.
SP Quant result databases store instruction fields as explicit columns. The
returned dataframe keeps those columns and adds a time_ns alias so existing
calculations and notebooks can use one timestamp name.
"""
validate_trading_instructions_table(conn)
columns = table_column_names(conn, "trading_instructions")
if "tstamp_ns" in columns:
select_expression = "*"
if "time_ns" not in columns:
select_expression = "*, tstamp_ns AS time_ns"
return pd.read_sql_query(
f"""
SELECT {select_expression}
FROM trading_instructions
ORDER BY tstamp_ns, rowid
""",
conn,
)
return pd.read_sql_query(
"SELECT * FROM trading_instructions ORDER BY time_ns, rowid",
conn,
@@ -429,6 +480,19 @@ def validate_ohlcv_1min_table(conn: sqlite3.Connection) -> None:
raise ValueError(f"ohlcv_1min is missing required column(s): {missing}")
def validate_market_table(conn: sqlite3.Connection) -> None:
"""Raise an actionable error if market lacks required columns."""
table_info = conn.execute("PRAGMA table_info(market)").fetchall()
if not table_info:
raise ValueError("SQLite database is missing required table: market")
existing_columns = {row[1] for row in table_info}
missing_columns = set(MARKET_COLUMNS) - existing_columns
if missing_columns:
missing = ", ".join(sorted(missing_columns))
raise ValueError(f"market is missing required column(s): {missing}")
def pair_assets_and_quote(pair_name: str) -> tuple[tuple[str, ...], str]:
"""Parse a pair name like ADA:USD-BTC:USD into assets and quote asset."""
pair_legs = pair_name.split("-")
@@ -454,6 +518,8 @@ def pair_assets_and_quote(pair_name: str) -> tuple[tuple[str, ...], str]:
def _parse_instruction_data(raw_data: Any) -> dict[str, Any] | None:
if raw_data is None:
return None
if isinstance(raw_data, dict):
return raw_data
try:
parsed = json.loads(raw_data)
@@ -463,6 +529,26 @@ def _parse_instruction_data(raw_data: Any) -> dict[str, Any] | None:
return parsed if isinstance(parsed, dict) else None
def _instruction_value(row: Any, column: str) -> Any:
return getattr(row, column, None)
def _instruction_payload(row: Any) -> dict[str, Any] | None:
legacy_payload = _parse_instruction_data(_instruction_value(row, "data"))
if legacy_payload is not None:
return legacy_payload
assets = _parse_instruction_data(_instruction_value(row, "assets"))
if assets is None:
return None
return {
"action": _instruction_value(row, "action"),
"quote_asset": _instruction_value(row, "quote_asset"),
"assets": assets,
}
def _finite_float(value: Any, field_name: str, pair_name: str) -> float:
if isinstance(value, bool):
raise ValueError(f"{field_name} for {pair_name} must be numeric, got bool")
@@ -527,15 +613,22 @@ def _matching_pair_instruction_rows(
pair_assets, quote_asset = pair_assets_and_quote(pair_name)
pair_asset_set = set(pair_assets)
if "data" not in trd_inst_df.columns:
raise ValueError("trading instructions dataframe is missing column: data")
legacy_columns = {"data"}
sp_quant_columns = set(SP_QUANT_TRADING_INSTRUCTIONS_COLUMNS)
dataframe_columns = set(trd_inst_df.columns)
if not (
legacy_columns <= dataframe_columns
or sp_quant_columns <= dataframe_columns
):
raise ValueError(
"trading instructions dataframe does not match a supported schema"
)
selected_instructions: list[dict[str, Any]] = []
for instruction_row in _sort_trading_instructions(trd_inst_df).itertuples(
index=False
):
raw_data = getattr(instruction_row, "data")
parsed = _parse_instruction_data(raw_data)
parsed = _instruction_payload(instruction_row)
if parsed is None or parsed.get("quote_asset") != quote_asset:
continue
@@ -547,6 +640,12 @@ def _matching_pair_instruction_rows(
{
"time_ns": getattr(instruction_row, "time_ns", None),
"tstamp": getattr(instruction_row, "tstamp", None),
"scaled_disequilibrium": getattr(
instruction_row,
"scaled_disequilibrium",
None,
),
"beta": getattr(instruction_row, "beta", None),
"data": parsed,
}
)
@@ -641,6 +740,10 @@ def calculate_pair_theo_executions(
"action": action,
"side": _execution_side(trade_size),
"strength": target_strength,
"scaled_disequilibrium": instruction[
"scaled_disequilibrium"
],
"beta": instruction["beta"],
"size": trade_size,
"price": price,
"usd_value": -trade_size * price,
@@ -671,6 +774,10 @@ def calculate_pair_theo_executions(
"action": action,
"side": _execution_side(trade_size),
"strength": None,
"scaled_disequilibrium": instruction[
"scaled_disequilibrium"
],
"beta": instruction["beta"],
"size": trade_size,
"price": price,
"usd_value": -trade_size * price,
@@ -689,6 +796,8 @@ def calculate_pair_theo_executions(
"action",
"side",
"strength",
"scaled_disequilibrium",
"beta",
"size",
"price",
"usd_value",
@@ -788,9 +897,9 @@ def load_pair_market_data(
*,
trading_day_start_ns: int,
) -> pd.DataFrame:
"""Load 1-minute close data from trading-day start for selected instruments."""
"""Load market close data from trading-day start for selected instruments."""
validate_selector_pair_instrument_columns(conn)
validate_ohlcv_1min_table(conn)
validate_market_table(conn)
pair_assets, _quote_asset = pair_assets_and_quote(pair_name)
selector_pair = pd.read_sql_query(
@@ -824,7 +933,7 @@ def load_pair_market_data(
exch_acct,
instrument_id,
close
FROM ohlcv_1min
FROM market
WHERE exch_acct = ? AND instrument_id = ? AND tstamp_ns >= ?
ORDER BY tstamp_ns, rowid
""",
@@ -842,7 +951,7 @@ def load_pair_market_data(
if missing_market_assets:
missing_assets = ", ".join(sorted(missing_market_assets))
raise ValueError(
f"ohlcv_1min does not contain market data for asset(s): {missing_assets}"
f"market does not contain data for asset(s): {missing_assets}"
)
market_data = pd.concat(market_frames, ignore_index=True)
@@ -854,7 +963,7 @@ def load_pair_market_data(
if missing_start_price_assets:
missing_assets = ", ".join(missing_start_price_assets)
raise ValueError(
"ohlcv_1min does not contain trading-day start close for "
"market does not contain trading-day start close for "
f"asset(s): {missing_assets}"
)
@@ -872,7 +981,7 @@ def load_pair_market_data(
sorted(market_data.loc[invalid_initial_close, "asset"].unique())
)
raise ValueError(
f"ohlcv_1min initial close must be positive for asset(s): {missing_assets}"
f"market initial close must be positive for asset(s): {missing_assets}"
)
market_data["relative_close"] = (