diff --git a/CHANGELOG.md b/CHANGELOG.md index 384aa32..c622469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,45 @@ All notable changes to this project are documented in this file. ## Unreleased -- No unreleased changes yet. +No unreleased changes yet. + +## 2026-07-28 v1.0.1 + +- Added the `spbt_day` notebook for interactive single-day backtest result + analysis, including SQLite result file selection from the local data + directory. +- Added selector-pair loading and dense ranking by `mr_score.final`, preserving + rows with invalid score JSON for inspection. +- Added theoretical return calculation for ranked pairs from + `trading_instructions`, including reusable helper functions and tests. +- Added a Plotly histogram for visual analysis of total theoretical return by + pair. +- Moved notebook support code into reusable `scripts/spbt_day.py` helpers. +- Adjusted notebook table outputs to show all relevant rows and reduce + redundant intermediate displays. +- Added an alphabetically sorted pair selector for individual pair analysis. +- Added selected-pair theoretical execution tables and aligned TheoRet + calculations with target-delta trade generation. +- Added per-asset `strength` values to selected-pair theoretical execution + tables. +- Corrected theoretical execution size to use + `10000 * strength / reference_price`. +- Removed `:USD` quote suffixes from displayed pair names in notebook tables, + chart hovers, and the pair selector dropdown while preserving full internal + pair keys for calculations. +- Added `num_trades` to pair TheoRet summaries, counting asset-level theoretical + trades from effective `TARGET` and `CLOSE` instructions. +- Added sortable interactive grids for the pair TheoRet and selected-pair + theoretical execution tables. +- Styled interactive dataframe grids with black text on white backgrounds for + readability across notebook themes. +- Added a selected-pair Plotly chart that overlays theoretical BUY/SELL + executions on relative 1-minute market close data for both instruments. +- Anchored the selected-pair market chart at trading-day midnight and normalized + relative prices to each instrument's close at that timestamp. +- Added a `min_pctg_change` threshold for ranked pair TheoRet calculations to + skip small target-strength changes after a position is acquired. +- Added a notebook input field for the minimum TARGET strength-change threshold. ## 2026-07-25 v0.0.9 diff --git a/notebooks/spbt_day.ipynb b/notebooks/spbt_day.ipynb index 4026182..daec8ec 100644 --- a/notebooks/spbt_day.ipynb +++ b/notebooks/spbt_day.ipynb @@ -23,39 +23,43 @@ "metadata": {}, "outputs": [], "source": [ - "from html import escape\n", "from pathlib import Path\n", - "import sqlite3\n", + "import importlib\n", "import sys\n", - "from urllib.parse import quote\n", "\n", "from IPython.display import display\n", "import ipywidgets as widgets\n", + "import pandas as pd\n", "\n", + "START_DIR = Path.cwd().resolve()\n", + "for candidate in (START_DIR, *START_DIR.parents):\n", + " if (candidate / \"scripts\" / \"spbt_day.py\").exists():\n", + " if str(candidate) not in sys.path:\n", + " sys.path.insert(0, str(candidate))\n", + " break\n", "\n", - "def find_repo_root(start: Path | None = None) -> Path:\n", - " \"\"\"Return the nearest parent containing repository-level files.\"\"\"\n", - " current = (start or Path.cwd()).resolve()\n", - " for candidate in (current, *current.parents):\n", - " if (candidate / \"requirements.txt\").exists() and (candidate / \"notebooks\").is_dir():\n", - " return candidate\n", - " return current\n", + "import scripts.spbt_day as spbt_day\n", "\n", + "spbt_day = importlib.reload(spbt_day)\n", + "\n", + "add_total_pnl = spbt_day.add_total_pnl\n", + "calculate_pair_theo_executions = spbt_day.calculate_pair_theo_executions\n", + "calculate_ranked_pairs_theo_ret = spbt_day.calculate_ranked_pairs_theo_ret\n", + "create_database_file_selector = spbt_day.create_database_file_selector\n", + "create_pair_name_dropdown = spbt_day.create_pair_name_dropdown\n", + "create_pair_trades_market_plot = spbt_day.create_pair_trades_market_plot\n", + "create_total_pnl_histogram = spbt_day.create_total_pnl_histogram\n", + "find_repo_root = spbt_day.find_repo_root\n", + "format_pair_name_for_display = spbt_day.format_pair_name_for_display\n", + "format_pair_names_for_display = spbt_day.format_pair_names_for_display\n", + "infer_trading_day_start_ns = spbt_day.infer_trading_day_start_ns\n", + "load_selector_pair_rankings = spbt_day.load_selector_pair_rankings\n", + "load_pair_market_data = spbt_day.load_pair_market_data\n", + "load_trading_instructions = spbt_day.load_trading_instructions\n", + "show_interactive_dataframe = spbt_day.show_interactive_dataframe\n", "\n", "REPO_ROOT = find_repo_root()\n", - "if str(REPO_ROOT) not in sys.path:\n", - " sys.path.insert(0, str(REPO_ROOT))\n", - "\n", - "from scripts.spbt_day import (\n", - " calculate_ranked_pairs_theo_ret,\n", - " load_selector_pair_rankings,\n", - " load_trading_instructions,\n", - ")\n", - "\n", "DEFAULT_DATA_DIR = REPO_ROOT / \"data\"\n", - "SQLITE_EXTENSIONS = {\".db\", \".sqlite\", \".sqlite3\"}\n", - "\n", - "selected_db_path: Path | None = None\n", "\n", "REPO_ROOT, DEFAULT_DATA_DIR" ] @@ -67,136 +71,12 @@ "metadata": {}, "outputs": [], "source": [ - "directory_input = widgets.Text(\n", - " value=str(DEFAULT_DATA_DIR),\n", - " description=\"Directory\",\n", - " continuous_update=False,\n", - " layout=widgets.Layout(width=\"100%\"),\n", - " style={\"description_width\": \"90px\"},\n", + "db_selector = create_database_file_selector(\n", + " default_data_dir=DEFAULT_DATA_DIR,\n", + " repo_root=REPO_ROOT,\n", ")\n", "\n", - "show_all_files = widgets.Checkbox(\n", - " value=False,\n", - " description=\"Show all files\",\n", - " indent=False,\n", - ")\n", - "\n", - "refresh_button = widgets.Button(\n", - " description=\"Refresh\",\n", - " icon=\"refresh\",\n", - " button_style=\"\",\n", - " tooltip=\"Rescan the selected directory\",\n", - ")\n", - "\n", - "file_select = widgets.Select(\n", - " options=[],\n", - " rows=12,\n", - " description=\"Files\",\n", - " layout=widgets.Layout(width=\"100%\"),\n", - " style={\"description_width\": \"90px\"},\n", - ")\n", - "\n", - "selected_path_display = widgets.HTML(value=\"Selected database: none\")\n", - "status_output = widgets.Output()\n", - "\n", - "\n", - "def normalize_directory(raw_path: str) -> Path:\n", - " path = Path(raw_path).expanduser()\n", - " if not path.is_absolute():\n", - " path = REPO_ROOT / path\n", - " return path.resolve()\n", - "\n", - "\n", - "def list_candidate_files(directory: Path, show_all: bool = False) -> list[Path]:\n", - " def result_file_sort_key(path: Path) -> tuple[int, str]:\n", - " name = path.name.lower()\n", - " if \".spbt_results.\" in name:\n", - " priority = 0\n", - " elif \"selector\" in name and \"results\" in name:\n", - " priority = 1\n", - " elif \"results\" in name:\n", - " priority = 2\n", - " else:\n", - " priority = 3\n", - " return priority, name\n", - "\n", - " if show_all:\n", - " return sorted(\n", - " (path for path in directory.iterdir() if path.is_file()),\n", - " key=result_file_sort_key,\n", - " )\n", - " return sorted(\n", - " (\n", - " path\n", - " for path in directory.iterdir()\n", - " if path.is_file() and path.suffix.lower() in SQLITE_EXTENSIONS\n", - " ),\n", - " key=result_file_sort_key,\n", - " )\n", - "\n", - "\n", - "def set_selected_database(path_value: str | None) -> None:\n", - " global selected_db_path\n", - " selected_db_path = Path(path_value).resolve() if path_value else None\n", - " label = str(selected_db_path) if selected_db_path else \"none\"\n", - " selected_path_display.value = f\"Selected database: {escape(label)}\"\n", - "\n", - "\n", - "def refresh_file_list(*_args) -> None:\n", - " directory = normalize_directory(directory_input.value)\n", - " with status_output:\n", - " status_output.clear_output()\n", - " if not directory.exists():\n", - " file_select.options = []\n", - " set_selected_database(None)\n", - " print(f\"Directory does not exist: {directory}\")\n", - " return\n", - " if not directory.is_dir():\n", - " file_select.options = []\n", - " set_selected_database(None)\n", - " print(f\"Path is not a directory: {directory}\")\n", - " return\n", - "\n", - " candidates = list_candidate_files(directory, show_all=show_all_files.value)\n", - " candidate_values = [str(path) for path in candidates]\n", - " previous_value = file_select.value\n", - " file_select.options = [(path.name, str(path)) for path in candidates]\n", - " if candidates:\n", - " file_select.value = previous_value if previous_value in candidate_values else candidate_values[0]\n", - " set_selected_database(file_select.value)\n", - " else:\n", - " set_selected_database(None)\n", - "\n", - " if candidates:\n", - " print(f\"Found {len(candidates)} file(s) in {directory}\")\n", - " else:\n", - " suffixes = \", \".join(sorted(SQLITE_EXTENSIONS))\n", - " print(f\"No SQLite files ({suffixes}) found in {directory}\")\n", - "\n", - "\n", - "def on_file_selected(change) -> None:\n", - " if change[\"name\"] == \"value\":\n", - " set_selected_database(change[\"new\"])\n", - "\n", - "\n", - "refresh_button.on_click(refresh_file_list)\n", - "show_all_files.observe(refresh_file_list, names=\"value\")\n", - "directory_input.observe(refresh_file_list, names=\"value\")\n", - "file_select.observe(on_file_selected, names=\"value\")\n", - "\n", - "display(\n", - " widgets.VBox(\n", - " [\n", - " widgets.HBox([directory_input, refresh_button]),\n", - " show_all_files,\n", - " file_select,\n", - " selected_path_display,\n", - " status_output,\n", - " ]\n", - " )\n", - ")\n", - "\n", - "refresh_file_list()" + "display(db_selector.widget)" ] }, { @@ -206,23 +86,8 @@ "metadata": {}, "outputs": [], "source": [ - "def selected_database_path() -> Path:\n", - " \"\"\"Return the interactively selected SQLite result path.\"\"\"\n", - " if selected_db_path is None:\n", - " raise ValueError(\"Choose a SQLite result file before continuing.\")\n", - " if not selected_db_path.exists():\n", - " raise FileNotFoundError(f\"Selected database does not exist: {selected_db_path}\")\n", - " if not selected_db_path.is_file():\n", - " raise ValueError(f\"Selected database path is not a file: {selected_db_path}\")\n", - " return selected_db_path\n", - "\n", - "\n", - "def connect_selected_database() -> sqlite3.Connection:\n", - " \"\"\"Open a read-only SQLite connection to the selected result database.\"\"\"\n", - " db_path = selected_database_path()\n", - " uri = f\"file:{quote(db_path.as_posix(), safe='/:')}?mode=ro\"\n", - " return sqlite3.connect(uri, uri=True)\n", - "\n", + "selected_database_path = db_selector.selected_database_path\n", + "connect_selected_database = db_selector.connect_selected_database\n", "\n", "# Later notebook sections can call selected_database_path() or connect_selected_database()." ] @@ -252,24 +117,11 @@ "finally:\n", " conn.close()\n", "\n", - "selector_pair_rankings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "selector-pair-ranking-summary", - "metadata": {}, - "outputs": [], - "source": [ - "selector_pair_ranking_summary = (\n", - " selector_pair_rankings[\"mr_score_parse_status\"]\n", - " .value_counts(dropna=False)\n", - " .rename_axis(\"mr_score_parse_status\")\n", - " .reset_index(name=\"row_count\")\n", + "selector_pair_rankings_display = format_pair_names_for_display(\n", + " selector_pair_rankings[[\"pair_rank\", \"pair_name\", \"mr_score_final\"]]\n", ")\n", - "\n", - "selector_pair_ranking_summary" + "with pd.option_context(\"display.max_rows\", None):\n", + " display(selector_pair_rankings_display)" ] }, { @@ -279,9 +131,26 @@ "source": [ "## Theoretical Return by Pair\n", "\n", - "Load `trading_instructions` and calculate theoretical return for each ranked pair. Each pair starts from a fixed `$10,000` theoretical USD base. `TARGET` opens or replaces the current theoretical position using each asset's `strength` and `reference_price`; `CLOSE` liquidates the open position at the close row's `reference_price`; `HOLD` is ignored.\n", + "Load `trading_instructions` and calculate theoretical return for each ranked pair. Each pair starts from a fixed `$10,000` theoretical USD base. `TARGET` trades from the current theoretical position to the new target position, where target size is `10000 * strength / reference_price`; `CLOSE` liquidates the open position at the close row's `reference_price`; `HOLD` is ignored.\n", "\n", - "`realized_pnl` and `unrealized_pnl` are percentage returns relative to `$10,000`, sorted by ascending MR rank." + "`MIN_TARGET_STRENGTH_CHANGE_PCTG` can be raised above `0.0` to skip `TARGET` updates whose absolute percentage strength change is smaller than the threshold since the position was acquired. `num_trades` counts asset-level theoretical trades caused by effective `TARGET` and `CLOSE` rows. `realized_pnl` and `unrealized_pnl` are percentage returns relative to `$10,000`. The displayed dataframe is sorted by total return (`realized_pnl + unrealized_pnl`) ascending." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "target-change-threshold-input", + "metadata": {}, + "outputs": [], + "source": [ + "min_target_change_input = widgets.FloatText(\n", + " value=0.0,\n", + " description=\"Mininal TARGET change (%)\",\n", + " step=1.0,\n", + " layout=widgets.Layout(width=\"420px\"),\n", + " style={\"description_width\": \"190px\"},\n", + ")\n", + "display(min_target_change_input)" ] }, { @@ -297,7 +166,7 @@ "finally:\n", " conn.close()\n", "\n", - "trading_instructions" + "print(f\"Loaded {len(trading_instructions):,} trading instruction rows.\")" ] }, { @@ -307,12 +176,160 @@ "metadata": {}, "outputs": [], "source": [ - "pair_theo_ret = calculate_ranked_pairs_theo_ret(\n", - " selector_pair_rankings,\n", + "MIN_TARGET_STRENGTH_CHANGE_PCTG = float(min_target_change_input.value)\n", + "\n", + "pair_theo_ret = add_total_pnl(\n", + " calculate_ranked_pairs_theo_ret(\n", + " selector_pair_rankings,\n", + " trading_instructions,\n", + " min_pctg_change=MIN_TARGET_STRENGTH_CHANGE_PCTG,\n", + " )\n", + ").sort_values(\n", + " [\"total_pnl\", \"pair_name\"],\n", + " ascending=[True, True],\n", + " kind=\"mergesort\",\n", + ").drop(columns=\"total_pnl\").reset_index(drop=True)\n", + "\n", + "pair_theo_ret_display = format_pair_names_for_display(pair_theo_ret)\n", + "\n", + "show_interactive_dataframe(\n", + " pair_theo_ret_display,\n", + " table_id=\"pair-theo-ret-grid\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "theoretical-return-histogram-context", + "metadata": {}, + "source": [ + "## Total Theoretical Return Distribution\n", + "\n", + "Plot the distribution of total theoretical return, calculated as `realized_pnl + unrealized_pnl`. Plotly chooses histogram bins automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "plot-total-theoretical-return-histogram", + "metadata": {}, + "outputs": [], + "source": [ + "total_pnl_histogram = create_total_pnl_histogram(pair_theo_ret)\n", + "\n", + "total_pnl_histogram" + ] + }, + { + "cell_type": "markdown", + "id": "individual-pair-analysis-context", + "metadata": {}, + "source": [ + "## Individual Pair Analysis\n", + "\n", + "Choose one pair for detailed follow-up analysis. Pair names are sorted alphabetically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "choose-individual-pair", + "metadata": {}, + "outputs": [], + "source": [ + "pair_name_dropdown = create_pair_name_dropdown(selector_pair_rankings)\n", + "display(pair_name_dropdown)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "selected-individual-pair", + "metadata": {}, + "outputs": [], + "source": [ + "selected_pair_name = pair_name_dropdown.value\n", + "format_pair_name_for_display(selected_pair_name)" + ] + }, + { + "cell_type": "markdown", + "id": "selected-pair-theo-executions-context", + "metadata": {}, + "source": [ + "### Selected Pair Theoretical Executions\n", + "\n", + "Create the theoretical asset-level executions used by the PnL calculation for the selected pair. `TARGET` rows trade the position difference from the current theoretical position to the new target position, where target size is `10000 * strength / reference_price`; `CLOSE` rows flatten the current theoretical position. Positive size is `BUY`; negative size is `SELL`; USD value is signed as the opposite cash movement." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "selected-pair-theo-executions", + "metadata": {}, + "outputs": [], + "source": [ + "selected_pair_theo_executions = calculate_pair_theo_executions(\n", + " selected_pair_name,\n", " trading_instructions,\n", + " min_pctg_change=MIN_TARGET_STRENGTH_CHANGE_PCTG,\n", ")\n", "\n", - "pair_theo_ret" + "selected_pair_theo_execution_columns = [\n", + " \"time\",\n", + " \"asset\",\n", + " \"action\",\n", + " \"side\",\n", + " \"strength\",\n", + " \"size\",\n", + " \"price\",\n", + " \"usd_value\",\n", + "]\n", + "selected_pair_theo_executions_display = selected_pair_theo_executions.reindex(\n", + " columns=selected_pair_theo_execution_columns\n", + ")\n", + "show_interactive_dataframe(\n", + " selected_pair_theo_executions_display,\n", + " table_id=\"selected-pair-theo-executions-grid\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "selected-pair-market-trades-context", + "metadata": {}, + "source": [ + "### Selected Pair Trades on Market Data\n", + "\n", + "Load full available 1-minute market data for the selected pair's instruments from `ohlcv_1min`, starting at midnight UTC of the trading day inferred from `trading_instructions`. Close prices are shown as relative prices from each instrument's close at that midnight. Theoretical executions are overlaid at their execution `reference_price`, normalized by the same midnight close. Execution markers use execution timestamps directly and do not require a matching OHLCV row." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "selected-pair-market-trades-plot", + "metadata": {}, + "outputs": [], + "source": [ + "trading_day_start_ns = infer_trading_day_start_ns(trading_instructions)\n", + "\n", + "conn = connect_selected_database()\n", + "try:\n", + " selected_pair_market_data = load_pair_market_data(\n", + " conn,\n", + " selected_pair_name,\n", + " trading_day_start_ns=trading_day_start_ns,\n", + " )\n", + "finally:\n", + " conn.close()\n", + "\n", + "selected_pair_market_trades_plot = create_pair_trades_market_plot(\n", + " selected_pair_name,\n", + " selected_pair_market_data,\n", + " selected_pair_theo_executions,\n", + ")\n", + "\n", + "selected_pair_market_trades_plot" ] } ], diff --git a/requirements.txt b/requirements.txt index d19d9d7..fd4658e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,11 @@ # Interactive analysis ipykernel>=6.29,<7 ipywidgets>=8.1,<9 +itables>=2.2,<3 jupyter>=1.1,<2 nbformat>=5.10,<6 pandas>=2.2,<3 +plotly>=5.24,<7 # Verification nbmake>=1.5,<2 diff --git a/scripts/spbt_day.py b/scripts/spbt_day.py index bd5082e..d467d60 100644 --- a/scripts/spbt_day.py +++ b/scripts/spbt_day.py @@ -2,17 +2,266 @@ from __future__ import annotations +from html import escape import json import math +from pathlib import Path import sqlite3 from typing import Any +from urllib.parse import quote import pandas as pd SELECTOR_PAIRS_COLUMNS = ("pair_name", "mr_score") +SELECTOR_PAIR_INSTRUMENT_COLUMNS = ("pair_name", "instrument_a", "instrument_b") TRADING_INSTRUCTIONS_COLUMNS = ("time_ns", "tstamp", "data") +OHLCV_1MIN_COLUMNS = ("tstamp", "tstamp_ns", "exch_acct", "instrument_id", "close") INITIAL_THEO_CAPITAL_USD = 10_000.0 +SQLITE_EXTENSIONS = {".db", ".sqlite", ".sqlite3"} +PAIR_NAME_DISPLAY_SUFFIX = ":USD" +INTERACTIVE_TABLE_CSS = """ +table.dataTable, +table.dataTable th, +table.dataTable td { + background-color: #ffffff !important; + color: #000000 !important; +} + +table.dataTable.display tbody tr.odd, +table.dataTable.display tbody tr.even, +table.dataTable.display tbody tr:hover, +table.dataTable.hover tbody tr:hover { + background-color: #ffffff !important; + color: #000000 !important; +} + +div.dt-container, +div.dt-container label, +div.dt-container input, +div.dt-container select, +div.dt-container .dt-info, +div.dt-container .dt-paging, +div.dt-container .dt-paging .dt-paging-button { + background-color: #ffffff !important; + color: #000000 !important; +} +""" + + +def find_repo_root(start: Path | None = None) -> Path: + """Return the nearest parent containing repository-level files.""" + current = (start or Path.cwd()).resolve() + for candidate in (current, *current.parents): + has_requirements = (candidate / "requirements.txt").exists() + has_notebooks = (candidate / "notebooks").is_dir() + if has_requirements and has_notebooks: + return candidate + return current + + +def normalize_directory(raw_path: str, base_dir: Path) -> Path: + """Resolve a user-provided directory path relative to a base directory.""" + path = Path(raw_path).expanduser() + if not path.is_absolute(): + path = base_dir / path + return path.resolve() + + +def result_file_sort_key(path: Path) -> tuple[int, str]: + """Sort result-like SQLite files before other local database files.""" + name = path.name.lower() + if ".spbt_results." in name: + priority = 0 + elif "selector" in name and "results" in name: + priority = 1 + elif "results" in name: + priority = 2 + else: + priority = 3 + return priority, name + + +def list_candidate_files(directory: Path, show_all: bool = False) -> list[Path]: + """List selectable files, preferring SQLite result databases by default.""" + if show_all: + candidates = (path for path in directory.iterdir() if path.is_file()) + else: + candidates = ( + path + for path in directory.iterdir() + if path.is_file() and path.suffix.lower() in SQLITE_EXTENSIONS + ) + return sorted(candidates, key=result_file_sort_key) + + +def read_only_sqlite_uri(db_path: Path) -> str: + """Build a read-only SQLite URI for a local database path.""" + return f"file:{quote(db_path.resolve().as_posix(), safe='/:')}?mode=ro" + + +def connect_sqlite_read_only(db_path: Path) -> sqlite3.Connection: + """Open a read-only SQLite connection for a local database path.""" + return sqlite3.connect(read_only_sqlite_uri(db_path), uri=True) + + +class DatabaseFileSelector: + """Small ipywidgets controller for choosing a local SQLite result database.""" + + def __init__( + self, + *, + repo_root: Path, + directory_input: Any, + show_all_files: Any, + refresh_button: Any, + file_select: Any, + selected_path_display: Any, + status_output: Any, + widget: Any, + ) -> None: + self.repo_root = repo_root + self.directory_input = directory_input + self.show_all_files = show_all_files + self.refresh_button = refresh_button + self.file_select = file_select + self.selected_path_display = selected_path_display + self.status_output = status_output + self.widget = widget + self.selected_db_path: Path | None = None + + def set_selected_database(self, path_value: str | None) -> None: + """Set the selected database and update its display label.""" + self.selected_db_path = Path(path_value).resolve() if path_value else None + label = str(self.selected_db_path) if self.selected_db_path else "none" + self.selected_path_display.value = f"Selected database: {escape(label)}" + + def refresh_file_list(self, *_args: Any) -> None: + """Rescan the configured directory and refresh selectable files.""" + directory = normalize_directory(self.directory_input.value, self.repo_root) + with self.status_output: + self.status_output.clear_output() + if not directory.exists(): + self.file_select.options = [] + self.set_selected_database(None) + print(f"Directory does not exist: {directory}") + return + if not directory.is_dir(): + self.file_select.options = [] + self.set_selected_database(None) + print(f"Path is not a directory: {directory}") + return + + candidates = list_candidate_files( + directory, + show_all=self.show_all_files.value, + ) + candidate_values = [str(path) for path in candidates] + previous_value = self.file_select.value + self.file_select.options = [(path.name, str(path)) for path in candidates] + if candidates: + self.file_select.value = ( + previous_value + if previous_value in candidate_values + else candidate_values[0] + ) + self.set_selected_database(self.file_select.value) + else: + self.set_selected_database(None) + + if candidates: + print(f"Found {len(candidates)} file(s) in {directory}") + else: + suffixes = ", ".join(sorted(SQLITE_EXTENSIONS)) + print(f"No SQLite files ({suffixes}) found in {directory}") + + def on_file_selected(self, change: dict[str, Any]) -> None: + """Update selected path when the widget selection changes.""" + if change["name"] == "value": + self.set_selected_database(change["new"]) + + def selected_database_path(self) -> Path: + """Return the interactively selected SQLite result path.""" + if self.selected_db_path is None: + raise ValueError("Choose a SQLite result file before continuing.") + if not self.selected_db_path.exists(): + raise FileNotFoundError( + f"Selected database does not exist: {self.selected_db_path}" + ) + if not self.selected_db_path.is_file(): + raise ValueError( + f"Selected database path is not a file: {self.selected_db_path}" + ) + return self.selected_db_path + + def connect_selected_database(self) -> sqlite3.Connection: + """Open a read-only SQLite connection to the selected result database.""" + return connect_sqlite_read_only(self.selected_database_path()) + + +def create_database_file_selector( + default_data_dir: Path | None = None, + repo_root: Path | None = None, +) -> DatabaseFileSelector: + """Create an interactive database file selector for notebook use.""" + import ipywidgets as widgets + + resolved_repo_root = (repo_root or find_repo_root()).resolve() + resolved_data_dir = (default_data_dir or resolved_repo_root / "data").resolve() + + directory_input = widgets.Text( + value=str(resolved_data_dir), + description="Directory", + continuous_update=False, + layout=widgets.Layout(width="100%"), + style={"description_width": "90px"}, + ) + show_all_files = widgets.Checkbox( + value=False, + description="Show all files", + indent=False, + ) + refresh_button = widgets.Button( + description="Refresh", + icon="refresh", + button_style="", + tooltip="Rescan the selected directory", + ) + file_select = widgets.Select( + options=[], + rows=12, + description="Files", + layout=widgets.Layout(width="100%"), + style={"description_width": "90px"}, + ) + selected_path_display = widgets.HTML(value="Selected database: none") + status_output = widgets.Output() + widget = widgets.VBox( + [ + widgets.HBox([directory_input, refresh_button]), + show_all_files, + file_select, + selected_path_display, + status_output, + ] + ) + + selector = DatabaseFileSelector( + repo_root=resolved_repo_root, + directory_input=directory_input, + show_all_files=show_all_files, + refresh_button=refresh_button, + file_select=file_select, + selected_path_display=selected_path_display, + status_output=status_output, + widget=widget, + ) + refresh_button.on_click(selector.refresh_file_list) + show_all_files.observe(selector.refresh_file_list, names="value") + directory_input.observe(selector.refresh_file_list, names="value") + file_select.observe(selector.on_file_selected, names="value") + selector.refresh_file_list() + return selector def parse_mr_score_final(raw_score: Any) -> tuple[float | None, str]: @@ -59,6 +308,19 @@ def validate_selector_pairs_table(conn: sqlite3.Connection) -> None: raise ValueError(f"selector_pairs is missing required column(s): {missing}") +def validate_selector_pair_instrument_columns(conn: sqlite3.Connection) -> None: + """Raise if selector_pairs cannot map pair names to market instruments.""" + table_info = conn.execute("PRAGMA table_info(selector_pairs)").fetchall() + if not table_info: + raise ValueError("SQLite database is missing required table: selector_pairs") + + existing_columns = {row[1] for row in table_info} + missing_columns = set(SELECTOR_PAIR_INSTRUMENT_COLUMNS) - existing_columns + if missing_columns: + missing = ", ".join(sorted(missing_columns)) + raise ValueError(f"selector_pairs is missing required column(s): {missing}") + + def rank_selector_pairs(selector_pairs: pd.DataFrame) -> pd.DataFrame: """Rank selector pairs by dense descending mr_score.final.""" missing_columns = set(SELECTOR_PAIRS_COLUMNS) - set(selector_pairs.columns) @@ -125,6 +387,32 @@ def load_trading_instructions(conn: sqlite3.Connection) -> pd.DataFrame: ) +def infer_trading_day_start_ns(trd_inst_df: pd.DataFrame) -> int: + """Infer the UTC midnight timestamp for the trading-instruction day.""" + if "time_ns" not in trd_inst_df.columns: + raise ValueError("trading instructions dataframe is missing column: time_ns") + + time_ns = pd.to_numeric(trd_inst_df["time_ns"], errors="coerce").dropna() + if time_ns.empty: + raise ValueError("trading instructions dataframe does not contain timestamps") + + first_timestamp = pd.to_datetime(int(time_ns.min()), unit="ns", utc=True) + return int(first_timestamp.floor("D").value) + + +def validate_ohlcv_1min_table(conn: sqlite3.Connection) -> None: + """Raise an actionable error if ohlcv_1min lacks required columns.""" + table_info = conn.execute("PRAGMA table_info(ohlcv_1min)").fetchall() + if not table_info: + raise ValueError("SQLite database is missing required table: ohlcv_1min") + + existing_columns = {row[1] for row in table_info} + missing_columns = set(OHLCV_1MIN_COLUMNS) - existing_columns + if missing_columns: + missing = ", ".join(sorted(missing_columns)) + raise ValueError(f"ohlcv_1min is missing required column(s): {missing}") + + def pair_assets_and_quote(pair_name: str) -> tuple[tuple[str, ...], str]: """Parse a pair name like ADA:USD-BTC:USD into assets and quote asset.""" pair_legs = pair_name.split("-") @@ -209,6 +497,16 @@ def _sort_trading_instructions(trd_inst_df: pd.DataFrame) -> pd.DataFrame: def _matching_pair_instructions( pair_name: str, trd_inst_df: pd.DataFrame, +) -> list[dict[str, Any]]: + return [ + instruction_row["data"] + for instruction_row in _matching_pair_instruction_rows(pair_name, trd_inst_df) + ] + + +def _matching_pair_instruction_rows( + pair_name: str, + trd_inst_df: pd.DataFrame, ) -> list[dict[str, Any]]: pair_assets, quote_asset = pair_assets_and_quote(pair_name) pair_asset_set = set(pair_assets) @@ -217,7 +515,10 @@ def _matching_pair_instructions( raise ValueError("trading instructions dataframe is missing column: data") selected_instructions: list[dict[str, Any]] = [] - for raw_data in _sort_trading_instructions(trd_inst_df)["data"]: + for instruction_row in _sort_trading_instructions(trd_inst_df).itertuples( + index=False + ): + raw_data = getattr(instruction_row, "data") parsed = _parse_instruction_data(raw_data) if parsed is None or parsed.get("quote_asset") != quote_asset: continue @@ -226,77 +527,500 @@ def _matching_pair_instructions( if not isinstance(assets, dict) or set(assets) != pair_asset_set: continue - selected_instructions.append(parsed) + selected_instructions.append( + { + "time_ns": getattr(instruction_row, "time_ns", None), + "tstamp": getattr(instruction_row, "tstamp", None), + "data": parsed, + } + ) return selected_instructions -def calculate_pair_theo_ret( +def _execution_side(size: float) -> str: + return "BUY" if size > 0 else "SELL" + + +def _validate_min_pctg_change(min_pctg_change: float) -> float: + try: + numeric_min_pctg_change = float(min_pctg_change) + except (TypeError, ValueError) as exc: + raise ValueError("min_pctg_change must be numeric") from exc + + if not math.isfinite(numeric_min_pctg_change): + raise ValueError("min_pctg_change must be finite") + if numeric_min_pctg_change < 0: + raise ValueError("min_pctg_change must be non-negative") + return numeric_min_pctg_change + + +def _target_strength_change_pctg( + current_strength: float | None, + target_strength: float, +) -> float | None: + if current_strength is None: + return None + if current_strength == 0: + return math.inf if target_strength != 0 else 0.0 + return abs((target_strength - current_strength) / current_strength) * 100.0 + + +def calculate_pair_theo_executions( pair_name: str, trd_inst_df: pd.DataFrame, -) -> dict[str, float | str]: - """Calculate realized and unrealized TheoRet percentages for one pair. + min_pctg_change: float = 0, +) -> pd.DataFrame: + """Create asset-level theoretical executions for one selected pair. - Repeated TARGET actions replace the previous open theoretical position. - CLOSE actions liquidate the currently open position. HOLD and unknown - actions are ignored. Returned PnL values are percentages of the fixed - 10,000 USD theoretical capital base. + TARGET rows trade from the current theoretical position to the new target + position when the absolute percentage strength change since the last + executed TARGET reaches min_pctg_change. CLOSE rows flatten the current + theoretical position. Positive size is a BUY; negative size is a SELL. USD + value is the opposite signed cash movement, so buys are negative and sells + are positive. """ + min_pctg_change = _validate_min_pctg_change(min_pctg_change) pair_assets, _quote_asset = pair_assets_and_quote(pair_name) - realized_pnl_usd = 0.0 - open_position: dict[str, dict[str, float]] | None = None + current_sizes = {asset: 0.0 for asset in pair_assets} + current_strengths: dict[str, float | None] = {asset: None for asset in pair_assets} + records: list[dict[str, Any]] = [] + execution_order = 0 - for instruction in _matching_pair_instructions(pair_name, trd_inst_df): - action = instruction.get("action") - assets_data = instruction["assets"] + for instruction in _matching_pair_instruction_rows(pair_name, trd_inst_df): + action = instruction["data"].get("action") + assets_data = instruction["data"]["assets"] if action == "TARGET": - open_position = {} for asset in pair_assets: asset_data = assets_data[asset] - quantity = INITIAL_THEO_CAPITAL_USD * _strength( + target_strength = _strength( asset_data, asset, pair_name, ) - entry_price = _reference_price(asset_data, asset, pair_name) - open_position[asset] = { - "quantity": quantity, - "entry_value": quantity * entry_price, - "latest_value": quantity * entry_price, - } + price = _reference_price(asset_data, asset, pair_name) + target_size = INITIAL_THEO_CAPITAL_USD * target_strength / price + strength_change_pctg = _target_strength_change_pctg( + current_strengths[asset], + target_strength, + ) + if ( + strength_change_pctg is not None + and strength_change_pctg < min_pctg_change + and not math.isclose(strength_change_pctg, min_pctg_change) + ): + continue + + trade_size = target_size - current_sizes[asset] + if trade_size == 0: + continue + + records.append( + { + "time": instruction["tstamp"] or instruction["time_ns"], + "time_ns": instruction["time_ns"], + "pair_name": pair_name, + "asset": asset, + "action": action, + "side": _execution_side(trade_size), + "strength": target_strength, + "size": trade_size, + "price": price, + "usd_value": -trade_size * price, + "_execution_order": execution_order, + } + ) + execution_order += 1 + current_sizes[asset] = target_size + current_strengths[asset] = target_strength elif action == "CLOSE": - if open_position is None: + if not any(current_sizes.values()): continue for asset in pair_assets: - asset_data = assets_data[asset] - close_value = ( - open_position[asset]["quantity"] - * _reference_price(asset_data, asset, pair_name) - ) - realized_pnl_usd += close_value - open_position[asset]["entry_value"] - open_position = None + current_size = current_sizes[asset] + if current_size == 0: + continue - unrealized_pnl_usd = 0.0 - if open_position is not None: - unrealized_pnl_usd = sum( - position["latest_value"] - position["entry_value"] - for position in open_position.values() + asset_data = assets_data[asset] + trade_size = -current_size + price = _reference_price(asset_data, asset, pair_name) + records.append( + { + "time": instruction["tstamp"] or instruction["time_ns"], + "time_ns": instruction["time_ns"], + "pair_name": pair_name, + "asset": asset, + "action": action, + "side": _execution_side(trade_size), + "strength": None, + "size": trade_size, + "price": price, + "usd_value": -trade_size * price, + "_execution_order": execution_order, + } + ) + execution_order += 1 + current_sizes[asset] = 0.0 + current_strengths[asset] = None + + columns = [ + "time", + "time_ns", + "pair_name", + "asset", + "action", + "side", + "strength", + "size", + "price", + "usd_value", + "_execution_order", + ] + return ( + pd.DataFrame.from_records(records, columns=columns) + .sort_values( + ["time_ns", "_execution_order"], + kind="mergesort", + na_position="last", ) + .drop(columns="_execution_order") + .reset_index(drop=True) + ) + + +def calculate_pair_theo_ret_from_executions( + pair_name: str, + trd_inst_df: pd.DataFrame, + min_pctg_change: float = 0, +) -> dict[str, float | int | str]: + """Calculate pair TheoRet from the generated theoretical executions.""" + executions = calculate_pair_theo_executions( + pair_name, + trd_inst_df, + min_pctg_change=min_pctg_change, + ) + if executions.empty: + return { + "pair_name": pair_name, + "num_trades": 0, + "realized_pnl": 0.0, + "unrealized_pnl": 0.0, + } + + pair_assets, _quote_asset = pair_assets_and_quote(pair_name) + current_sizes = {asset: 0.0 for asset in pair_assets} + latest_prices = {asset: 0.0 for asset in pair_assets} + open_cash_flow_usd = 0.0 + realized_pnl_usd = 0.0 + + for execution in executions.itertuples(index=False): + current_sizes[execution.asset] += execution.size + latest_prices[execution.asset] = execution.price + open_cash_flow_usd += execution.usd_value + if execution.action == "CLOSE" and not any(current_sizes.values()): + realized_pnl_usd += open_cash_flow_usd + open_cash_flow_usd = 0.0 + + unrealized_pnl_usd = open_cash_flow_usd + sum( + current_sizes[asset] * latest_prices[asset] for asset in pair_assets + ) return { "pair_name": pair_name, + "num_trades": len(executions), "realized_pnl": realized_pnl_usd / INITIAL_THEO_CAPITAL_USD * 100.0, "unrealized_pnl": unrealized_pnl_usd / INITIAL_THEO_CAPITAL_USD * 100.0, } +def calculate_pair_theo_ret( + pair_name: str, + trd_inst_df: pd.DataFrame, + min_pctg_change: float = 0, +) -> dict[str, float | int | str]: + """Calculate realized and unrealized TheoRet percentages for one pair. + + TARGET actions trade the delta between current and target theoretical + positions when the target strength change reaches min_pctg_change. CLOSE + actions liquidate the currently open position. HOLD and unknown actions are + ignored. Returned PnL values are percentages of the fixed 10,000 USD + theoretical capital base. + """ + return calculate_pair_theo_ret_from_executions( + pair_name, + trd_inst_df, + min_pctg_change=min_pctg_change, + ) + + +def parse_selector_instrument(raw_instrument: Any) -> tuple[str, str]: + """Parse selector_pairs instrument value into exch_acct and instrument_id.""" + if not isinstance(raw_instrument, str) or ":" not in raw_instrument: + raise ValueError(f"selector instrument must use EXCH_ACCT:INSTRUMENT form") + + exch_acct, instrument_id = raw_instrument.split(":", 1) + if not exch_acct or not instrument_id: + raise ValueError(f"selector instrument must use EXCH_ACCT:INSTRUMENT form") + return exch_acct, instrument_id + + +def load_pair_market_data( + conn: sqlite3.Connection, + pair_name: str, + *, + trading_day_start_ns: int, +) -> pd.DataFrame: + """Load 1-minute close data from trading-day start for selected instruments.""" + validate_selector_pair_instrument_columns(conn) + validate_ohlcv_1min_table(conn) + pair_assets, _quote_asset = pair_assets_and_quote(pair_name) + + selector_pair = pd.read_sql_query( + """ + SELECT instrument_a, instrument_b + FROM selector_pairs + WHERE pair_name = ? + ORDER BY rowid + LIMIT 1 + """, + conn, + params=(pair_name,), + ) + if selector_pair.empty: + raise ValueError(f"selector_pairs does not contain pair_name: {pair_name}") + + instrument_values = [ + selector_pair["instrument_a"].iloc[0], + selector_pair["instrument_b"].iloc[0], + ] + + market_frames = [] + missing_market_assets = [] + for asset, raw_instrument in zip(pair_assets, instrument_values, strict=True): + exch_acct, instrument_id = parse_selector_instrument(raw_instrument) + instrument_market_data = pd.read_sql_query( + """ + SELECT + tstamp AS time, + tstamp_ns AS time_ns, + exch_acct, + instrument_id, + close + FROM ohlcv_1min + WHERE exch_acct = ? AND instrument_id = ? AND tstamp_ns >= ? + ORDER BY tstamp_ns, rowid + """, + conn, + params=(exch_acct, instrument_id, trading_day_start_ns), + ) + if instrument_market_data.empty: + missing_market_assets.append(asset) + continue + + instrument_market_data["pair_name"] = pair_name + instrument_market_data["asset"] = asset + market_frames.append(instrument_market_data) + + if missing_market_assets: + missing_assets = ", ".join(sorted(missing_market_assets)) + raise ValueError( + f"ohlcv_1min does not contain market data for asset(s): {missing_assets}" + ) + + market_data = pd.concat(market_frames, ignore_index=True) + market_data["close"] = pd.to_numeric(market_data["close"], errors="coerce") + missing_start_price_assets = sorted( + set(pair_assets) + - set(market_data.loc[market_data["time_ns"] == trading_day_start_ns, "asset"]) + ) + if missing_start_price_assets: + missing_assets = ", ".join(missing_start_price_assets) + raise ValueError( + "ohlcv_1min does not contain trading-day start close for " + f"asset(s): {missing_assets}" + ) + + initial_close_by_asset = market_data.drop_duplicates("asset").set_index("asset")[ + "close" + ] + market_data["initial_close"] = market_data["asset"].map( + initial_close_by_asset + ) + invalid_initial_close = ( + market_data["initial_close"].isna() | (market_data["initial_close"] <= 0) + ) + if invalid_initial_close.any(): + missing_assets = ", ".join( + sorted(market_data.loc[invalid_initial_close, "asset"].unique()) + ) + raise ValueError( + f"ohlcv_1min initial close must be positive for asset(s): {missing_assets}" + ) + + market_data["relative_close"] = ( + market_data["close"] - market_data["initial_close"] + ) / market_data["initial_close"] + return market_data.loc[ + :, + [ + "pair_name", + "asset", + "time", + "time_ns", + "exch_acct", + "instrument_id", + "close", + "initial_close", + "relative_close", + ], + ] + + +def _normalize_trade_prices_to_initial_close( + theo_executions: pd.DataFrame, + market_data: pd.DataFrame, +) -> pd.DataFrame: + required_execution_columns = {"asset", "side", "price", "time", "time_ns"} + missing_execution_columns = required_execution_columns - set( + theo_executions.columns + ) + if missing_execution_columns: + missing = ", ".join(sorted(missing_execution_columns)) + raise ValueError(f"theo executions dataframe missing column(s): {missing}") + + required_market_columns = {"asset", "initial_close"} + missing_market_columns = required_market_columns - set(market_data.columns) + if missing_market_columns: + missing = ", ".join(sorted(missing_market_columns)) + raise ValueError(f"market data dataframe missing column(s): {missing}") + + if theo_executions.empty: + return theo_executions.assign(relative_price=pd.Series(dtype="float64")) + + initial_close_by_asset = ( + market_data.dropna(subset=["initial_close"]) + .drop_duplicates("asset") + .set_index("asset")["initial_close"] + ) + trades = theo_executions.copy() + trades["initial_close"] = trades["asset"].map(initial_close_by_asset) + missing_initial_close = trades["initial_close"].isna() | ( + trades["initial_close"] <= 0 + ) + if missing_initial_close.any(): + missing_assets = ", ".join( + sorted(trades.loc[missing_initial_close, "asset"].dropna().unique()) + ) + raise ValueError( + f"market data initial close is required for trade asset(s): {missing_assets}" + ) + + trades["price"] = pd.to_numeric(trades["price"], errors="coerce") + trades["relative_price"] = ( + trades["price"] - trades["initial_close"] + ) / trades["initial_close"] + return trades + + +def create_pair_trades_market_plot( + pair_name: str, + market_data: pd.DataFrame, + theo_executions: pd.DataFrame, +) -> Any: + """Plot relative market closes and selected-pair theoretical trades.""" + import plotly.graph_objects as go + + required_market_columns = {"asset", "time", "close", "relative_close"} + missing_market_columns = required_market_columns - set(market_data.columns) + if missing_market_columns: + missing = ", ".join(sorted(missing_market_columns)) + raise ValueError(f"market data dataframe missing column(s): {missing}") + + pair_assets, _quote_asset = pair_assets_and_quote(pair_name) + trades = _normalize_trade_prices_to_initial_close(theo_executions, market_data) + figure = go.Figure() + + for asset in pair_assets: + asset_market_data = market_data.loc[market_data["asset"] == asset].sort_values( + ["time_ns", "time"], + kind="mergesort", + na_position="last", + ) + figure.add_trace( + go.Scatter( + x=asset_market_data["time"], + y=asset_market_data["relative_close"], + mode="lines", + name=f"{asset} close", + hovertemplate=( + "Asset=%{customdata[0]}
" + "Time=%{x}
" + "Close=%{customdata[1]:.8g}
" + "Relative=%{y:.4%}" + ), + customdata=asset_market_data[["asset", "close"]], + ) + ) + + for side, color, symbol in ( + ("BUY", "darkgreen", "triangle-up"), + ("SELL", "darkred", "triangle-down"), + ): + asset_side_trades = trades.loc[ + (trades["asset"] == asset) & (trades["side"] == side) + ].sort_values(["time_ns", "time"], kind="mergesort", na_position="last") + if asset_side_trades.empty: + continue + + figure.add_trace( + go.Scatter( + x=asset_side_trades["time"], + y=asset_side_trades["relative_price"], + mode="markers", + name=f"{asset} {side}", + marker={ + "symbol": symbol, + "color": color, + "size": 11, + "line": {"color": "white", "width": 1}, + }, + hovertemplate=( + "Asset=%{customdata[0]}
" + "Side=%{customdata[1]}
" + "Action=%{customdata[2]}
" + "Time=%{x}
" + "Price=%{customdata[3]:.8g}
" + "Relative=%{y:.4%}
" + "Size=%{customdata[4]:.8g}" + ), + customdata=asset_side_trades[ + ["asset", "side", "action", "price", "size"] + ], + ) + ) + + figure.update_layout( + title=f"{format_pair_name_for_display(pair_name)} Trades on Market Data", + xaxis_title="Time", + yaxis_title="Relative price", + hovermode="x unified", + legend_title="Series", + ) + if not market_data.empty: + figure.update_xaxes(range=[market_data["time"].min(), market_data["time"].max()]) + figure.update_yaxes(tickformat=".2%") + return figure + + def calculate_ranked_pairs_theo_ret( selector_pair_rankings: pd.DataFrame, trd_inst_df: pd.DataFrame, + min_pctg_change: float = 0, ) -> pd.DataFrame: """Calculate TheoRet percentages for every ranked selector pair.""" + min_pctg_change = _validate_min_pctg_change(min_pctg_change) required_columns = {"pair_name", "pair_rank"} missing_columns = required_columns - set(selector_pair_rankings.columns) if missing_columns: @@ -305,11 +1029,16 @@ def calculate_ranked_pairs_theo_ret( records = [] for row in selector_pair_rankings.itertuples(index=False): - pair_result = calculate_pair_theo_ret(row.pair_name, trd_inst_df) + pair_result = calculate_pair_theo_ret( + row.pair_name, + trd_inst_df, + min_pctg_change=min_pctg_change, + ) records.append( { "pair_name": pair_result["pair_name"], "mr_ranking": row.pair_rank, + "num_trades": pair_result["num_trades"], "realized_pnl": pair_result["realized_pnl"], "unrealized_pnl": pair_result["unrealized_pnl"], } @@ -318,8 +1047,131 @@ def calculate_ranked_pairs_theo_ret( return ( pd.DataFrame.from_records( records, - columns=["pair_name", "mr_ranking", "realized_pnl", "unrealized_pnl"], + columns=[ + "pair_name", + "mr_ranking", + "num_trades", + "realized_pnl", + "unrealized_pnl", + ], ) .sort_values(["mr_ranking", "pair_name"], na_position="last", kind="mergesort") .reset_index(drop=True) ) + + +def format_pair_name_for_display(pair_name: Any) -> Any: + """Return a human-facing pair label without the USD quote suffix.""" + if pd.isna(pair_name): + return pair_name + return "-".join( + leg.removesuffix(PAIR_NAME_DISPLAY_SUFFIX) for leg in str(pair_name).split("-") + ) + + +def format_pair_names_for_display( + dataframe: pd.DataFrame, + column: str = "pair_name", +) -> pd.DataFrame: + """Return a copy with pair-name labels formatted for display.""" + if column not in dataframe.columns: + raise ValueError(f"dataframe missing column: {column}") + formatted = dataframe.copy() + formatted[column] = formatted[column].map(format_pair_name_for_display) + return formatted + + +def add_total_pnl(pair_theo_ret: pd.DataFrame) -> pd.DataFrame: + """Return a copy of pair TheoRet rows with total realized plus unrealized PnL.""" + required_columns = {"realized_pnl", "unrealized_pnl"} + missing_columns = required_columns - set(pair_theo_ret.columns) + if missing_columns: + missing = ", ".join(sorted(missing_columns)) + raise ValueError(f"pair TheoRet dataframe missing column(s): {missing}") + return pair_theo_ret.assign( + total_pnl=pair_theo_ret["realized_pnl"] + pair_theo_ret["unrealized_pnl"] + ) + + +def create_total_pnl_histogram(pair_theo_ret: pd.DataFrame) -> Any: + """Create a Plotly histogram of total theoretical return with automatic bins.""" + import plotly.express as px + + pair_theo_ret_for_plot = format_pair_names_for_display(add_total_pnl(pair_theo_ret)) + hover_columns = [ + column + for column in ("pair_name", "mr_ranking", "realized_pnl", "unrealized_pnl") + if column in pair_theo_ret_for_plot.columns + ] + total_pnl_histogram = px.histogram( + pair_theo_ret_for_plot, + x="total_pnl", + labels={ + "total_pnl": "Total TheoRet (%)", + "count": "Pair count", + }, + title="Total TheoRet Distribution by Pair", + hover_data=hover_columns, + ) + total_pnl_histogram.update_layout( + bargap=0.05, + yaxis_title="Pair count", + ) + return total_pnl_histogram + + +def show_interactive_dataframe( + dataframe: pd.DataFrame, + *, + table_id: str | None = None, + **kwargs: Any, +) -> None: + """Render a dataframe as an interactive sortable notebook grid.""" + from itables import show + + options = { + "paging": True, + "pageLength": 25, + "scrollX": True, + "ordering": True, + "showIndex": False, + "maxBytes": "8MB", + "classes": "display compact stripe hover", + "css": INTERACTIVE_TABLE_CSS, + } + if table_id is not None: + options["table_id"] = table_id + options.update(kwargs) + show(dataframe, **options) + + +def sorted_pair_names(selector_pair_rankings: pd.DataFrame) -> list[str]: + """Return unique pair names sorted alphabetically for pair-level analysis.""" + if "pair_name" not in selector_pair_rankings.columns: + raise ValueError("selector pair rankings missing column: pair_name") + return sorted( + { + str(pair_name) + for pair_name in selector_pair_rankings["pair_name"].dropna() + if str(pair_name) + } + ) + + +def create_pair_name_dropdown(selector_pair_rankings: pd.DataFrame) -> Any: + """Create a dropdown for choosing one pair name from ranked pairs.""" + import ipywidgets as widgets + + pair_names = sorted_pair_names(selector_pair_rankings) + if not pair_names: + raise ValueError("selector pair rankings do not contain any pair names") + options = [ + (format_pair_name_for_display(pair_name), pair_name) for pair_name in pair_names + ] + return widgets.Dropdown( + options=options, + value=pair_names[0], + description="Pair", + layout=widgets.Layout(width="100%"), + style={"description_width": "90px"}, + ) diff --git a/tests/test_spbt_day.py b/tests/test_spbt_day.py index 84a28c0..1a6cf26 100644 --- a/tests/test_spbt_day.py +++ b/tests/test_spbt_day.py @@ -1,16 +1,34 @@ import sqlite3 +from pathlib import Path import pandas as pd import pytest from scripts.spbt_day import ( + add_total_pnl, + calculate_pair_theo_executions, calculate_pair_theo_ret, calculate_ranked_pairs_theo_ret, + create_pair_name_dropdown, + create_pair_trades_market_plot, + connect_sqlite_read_only, + create_total_pnl_histogram, + find_repo_root, + format_pair_name_for_display, + format_pair_names_for_display, + infer_trading_day_start_ns, + list_candidate_files, + load_pair_market_data, load_selector_pair_rankings, load_trading_instructions, + normalize_directory, pair_assets_and_quote, + parse_selector_instrument, parse_mr_score_final, rank_selector_pairs, + read_only_sqlite_uri, + show_interactive_dataframe, + sorted_pair_names, ) @@ -89,6 +107,318 @@ def test_pair_assets_and_quote_parses_two_leg_pair(): assert pair_assets_and_quote("ADA:USD-BTC:USD") == (("ADA", "BTC"), "USD") +def test_infer_trading_day_start_ns_uses_utc_midnight(): + trd_inst_df = pd.DataFrame( + { + "time_ns": [ + pd.Timestamp("2026-06-17T02:25:00Z").value, + pd.Timestamp("2026-06-17T00:01:00Z").value, + ] + } + ) + + assert infer_trading_day_start_ns(trd_inst_df) == pd.Timestamp( + "2026-06-17T00:00:00Z" + ).value + + +def test_parse_selector_instrument_splits_exchange_account_and_instrument_id(): + assert parse_selector_instrument("COINBASE_AT:PAIR-ADA-USD") == ( + "COINBASE_AT", + "PAIR-ADA-USD", + ) + + +def test_load_pair_market_data_maps_selector_instruments_and_relative_close(): + trading_day_start_ns = 10 + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE selector_pairs ( + pair_name TEXT, + instrument_a TEXT, + instrument_b TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE ohlcv_1min ( + tstamp TEXT, + tstamp_ns INTEGER, + exch_acct TEXT, + instrument_id TEXT, + close REAL + ) + """ + ) + conn.execute( + "INSERT INTO selector_pairs VALUES (?, ?, ?)", + ( + "AAA:USD-BBB:USD", + "EXCH_A:PAIR-AAA-USD", + "EXCH_B:PAIR-BBB-USD", + ), + ) + conn.executemany( + "INSERT INTO ohlcv_1min VALUES (?, ?, ?, ?, ?)", + [ + ("pre", 9, "EXCH_A", "PAIR-AAA-USD", 90.0), + ("t0", 10, "EXCH_A", "PAIR-AAA-USD", 100.0), + ("t1", 11, "EXCH_A", "PAIR-AAA-USD", 110.0), + ("pre", 9, "EXCH_B", "PAIR-BBB-USD", 55.0), + ("t0", 10, "EXCH_B", "PAIR-BBB-USD", 50.0), + ("t1", 11, "EXCH_B", "PAIR-BBB-USD", 45.0), + ("t0", 10, "OTHER", "PAIR-AAA-USD", 999.0), + ], + ) + + market_data = load_pair_market_data( + conn, + "AAA:USD-BBB:USD", + trading_day_start_ns=trading_day_start_ns, + ) + + assert market_data[ + ["asset", "exch_acct", "instrument_id", "close", "initial_close"] + ].to_dict("records") == [ + { + "asset": "AAA", + "exch_acct": "EXCH_A", + "instrument_id": "PAIR-AAA-USD", + "close": 100.0, + "initial_close": 100.0, + }, + { + "asset": "AAA", + "exch_acct": "EXCH_A", + "instrument_id": "PAIR-AAA-USD", + "close": 110.0, + "initial_close": 100.0, + }, + { + "asset": "BBB", + "exch_acct": "EXCH_B", + "instrument_id": "PAIR-BBB-USD", + "close": 50.0, + "initial_close": 50.0, + }, + { + "asset": "BBB", + "exch_acct": "EXCH_B", + "instrument_id": "PAIR-BBB-USD", + "close": 45.0, + "initial_close": 50.0, + }, + ] + assert market_data["time_ns"].tolist() == [10, 11, 10, 11] + assert market_data["relative_close"].tolist() == [ + 0.0, + pytest.approx(0.1), + 0.0, + pytest.approx(-0.1), + ] + + +def test_load_pair_market_data_requires_market_rows_for_both_assets(): + trading_day_start_ns = 1 + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE selector_pairs ( + pair_name TEXT, + instrument_a TEXT, + instrument_b TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE ohlcv_1min ( + tstamp TEXT, + tstamp_ns INTEGER, + exch_acct TEXT, + instrument_id TEXT, + close REAL + ) + """ + ) + conn.execute( + "INSERT INTO selector_pairs VALUES (?, ?, ?)", + ( + "AAA:USD-BBB:USD", + "EXCH_A:PAIR-AAA-USD", + "EXCH_B:PAIR-BBB-USD", + ), + ) + conn.execute( + "INSERT INTO ohlcv_1min VALUES (?, ?, ?, ?, ?)", + ("t1", 1, "EXCH_A", "PAIR-AAA-USD", 100.0), + ) + + with pytest.raises( + ValueError, + match=r"ohlcv_1min does not contain market data for asset\(s\): BBB", + ): + load_pair_market_data( + conn, + "AAA:USD-BBB:USD", + trading_day_start_ns=trading_day_start_ns, + ) + + +def test_load_pair_market_data_requires_time_zero_close_for_each_asset(): + trading_day_start_ns = 1 + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE selector_pairs ( + pair_name TEXT, + instrument_a TEXT, + instrument_b TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE ohlcv_1min ( + tstamp TEXT, + tstamp_ns INTEGER, + exch_acct TEXT, + instrument_id TEXT, + close REAL + ) + """ + ) + conn.execute( + "INSERT INTO selector_pairs VALUES (?, ?, ?)", + ( + "AAA:USD-BBB:USD", + "EXCH_A:PAIR-AAA-USD", + "EXCH_B:PAIR-BBB-USD", + ), + ) + conn.executemany( + "INSERT INTO ohlcv_1min VALUES (?, ?, ?, ?, ?)", + [ + ("t1", 1, "EXCH_A", "PAIR-AAA-USD", None), + ("t2", 2, "EXCH_A", "PAIR-AAA-USD", 110.0), + ("t1", 1, "EXCH_B", "PAIR-BBB-USD", 50.0), + ], + ) + + with pytest.raises( + ValueError, + match=r"ohlcv_1min initial close must be positive for asset\(s\): AAA", + ): + load_pair_market_data( + conn, + "AAA:USD-BBB:USD", + trading_day_start_ns=trading_day_start_ns, + ) + + +def test_load_pair_market_data_requires_exact_trading_day_start_row(): + trading_day_start_ns = 10 + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE selector_pairs ( + pair_name TEXT, + instrument_a TEXT, + instrument_b TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE ohlcv_1min ( + tstamp TEXT, + tstamp_ns INTEGER, + exch_acct TEXT, + instrument_id TEXT, + close REAL + ) + """ + ) + conn.execute( + "INSERT INTO selector_pairs VALUES (?, ?, ?)", + ( + "AAA:USD-BBB:USD", + "EXCH_A:PAIR-AAA-USD", + "EXCH_B:PAIR-BBB-USD", + ), + ) + conn.executemany( + "INSERT INTO ohlcv_1min VALUES (?, ?, ?, ?, ?)", + [ + ("t1", 11, "EXCH_A", "PAIR-AAA-USD", 110.0), + ("t0", 10, "EXCH_B", "PAIR-BBB-USD", 50.0), + ], + ) + + with pytest.raises( + ValueError, + match=( + "ohlcv_1min does not contain trading-day start close for " + r"asset\(s\): AAA" + ), + ): + load_pair_market_data( + conn, + "AAA:USD-BBB:USD", + trading_day_start_ns=trading_day_start_ns, + ) + + +def test_create_pair_trades_market_plot_adds_relative_lines_and_trade_markers(): + market_data = pd.DataFrame( + { + "pair_name": ["AAA:USD-BBB:USD"] * 4, + "asset": ["AAA", "AAA", "BBB", "BBB"], + "time": ["t1", "t2", "t1", "t2"], + "time_ns": [1, 2, 1, 2], + "close": [100.0, 110.0, 50.0, 45.0], + "initial_close": [100.0, 100.0, 50.0, 50.0], + "relative_close": [0.0, 0.1, 0.0, -0.1], + } + ) + theo_executions = pd.DataFrame( + { + "time": ["t1.5", "t2.5"], + "time_ns": [15, 25], + "asset": ["AAA", "BBB"], + "action": ["TARGET", "CLOSE"], + "side": ["BUY", "SELL"], + "size": [2.0, -3.0], + "price": [105.0, 40.0], + } + ) + + figure = create_pair_trades_market_plot( + "AAA:USD-BBB:USD", + market_data, + theo_executions, + ) + + assert [trace.name for trace in figure.data] == [ + "AAA close", + "AAA BUY", + "BBB close", + "BBB SELL", + ] + assert figure.data[0].y.tolist() == [0.0, 0.1] + assert figure.data[1].marker.symbol == "triangle-up" + assert figure.data[1].marker.color == "darkgreen" + assert figure.data[1].x.tolist() == ["t1.5"] + assert figure.data[1].y.tolist() == [pytest.approx(0.05)] + assert figure.data[3].marker.symbol == "triangle-down" + assert figure.data[3].marker.color == "darkred" + assert figure.data[3].x.tolist() == ["t2.5"] + assert figure.data[3].y.tolist() == [pytest.approx(-0.2)] + assert figure.layout.xaxis.range == ("t1", "t2") + + def test_calculate_pair_theo_ret_replaces_targets_and_closes_open_position(): trd_inst_df = pd.DataFrame( { @@ -118,7 +448,297 @@ def test_calculate_pair_theo_ret_replaces_targets_and_closes_open_position(): assert theo_ret == { "pair_name": "AAA:USD-BBB:USD", - "realized_pnl": pytest.approx(24.0), + "num_trades": 6, + "realized_pnl": pytest.approx(0.1), + "unrealized_pnl": 0.0, + } + + +def test_calculate_pair_theo_executions_uses_target_deltas_and_signed_cash(): + trd_inst_df = pd.DataFrame( + { + "time_ns": [1, 2, 3, 4], + "tstamp": ["t1", "t2", "t3", "t4"], + "data": [ + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"95"},' + '"BBB":{"reference_price":"55"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.01"},' + '"BBB":{"reference_price":"50","strength":"-0.02"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"110","strength":"0.015"},' + '"BBB":{"reference_price":"45","strength":"-0.01"}}}' + ), + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"120"},' + '"BBB":{"reference_price":"40"}}}' + ), + ], + } + ) + + executions = calculate_pair_theo_executions("AAA:USD-BBB:USD", trd_inst_df) + + display_columns = [ + "time", + "asset", + "action", + "side", + "strength", + "size", + "price", + "usd_value", + ] + execution_rows = executions[display_columns].to_dict("records") + + assert execution_rows[:4] == [ + { + "time": "t2", + "asset": "AAA", + "action": "TARGET", + "side": "BUY", + "strength": 0.01, + "size": 1.0, + "price": 100.0, + "usd_value": -100.0, + }, + { + "time": "t2", + "asset": "BBB", + "action": "TARGET", + "side": "SELL", + "strength": -0.02, + "size": -4.0, + "price": 50.0, + "usd_value": 200.0, + }, + { + "time": "t3", + "asset": "AAA", + "action": "TARGET", + "side": "BUY", + "strength": 0.015, + "size": pytest.approx(0.36363636363636365), + "price": 110.0, + "usd_value": pytest.approx(-40.0), + }, + { + "time": "t3", + "asset": "BBB", + "action": "TARGET", + "side": "BUY", + "strength": -0.01, + "size": pytest.approx(1.7777777777777777), + "price": 45.0, + "usd_value": pytest.approx(-80.0), + }, + ] + assert execution_rows[4] | {"strength": None} == { + "time": "t4", + "asset": "AAA", + "action": "CLOSE", + "side": "SELL", + "strength": None, + "size": pytest.approx(-1.3636363636363638), + "price": 120.0, + "usd_value": pytest.approx(163.63636363636365), + } + assert execution_rows[5] | {"strength": None} == { + "time": "t4", + "asset": "BBB", + "action": "CLOSE", + "side": "BUY", + "strength": None, + "size": pytest.approx(2.2222222222222223), + "price": 40.0, + "usd_value": pytest.approx(-88.88888888888889), + } + assert executions["strength"].iloc[:4].tolist() == [0.01, -0.02, 0.015, -0.01] + assert executions["strength"].iloc[4:].isna().all() + + +def test_calculate_pair_theo_executions_skips_small_target_strength_changes(): + trd_inst_df = pd.DataFrame( + { + "time_ns": [1, 2, 3, 4], + "tstamp": ["t1", "t2", "t3", "t4"], + "data": [ + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.5"},' + '"BBB":{"reference_price":"50","strength":"-0.5"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.53"},' + '"BBB":{"reference_price":"50","strength":"-0.47"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.7"},' + '"BBB":{"reference_price":"50","strength":"-0.7"}}}' + ), + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100"},' + '"BBB":{"reference_price":"50"}}}' + ), + ], + } + ) + + executions = calculate_pair_theo_executions( + "AAA:USD-BBB:USD", + trd_inst_df, + min_pctg_change=25, + ) + + execution_rows = executions[["time", "asset", "action", "strength", "size"]] + + assert execution_rows.iloc[:4].to_dict("records") == [ + { + "time": "t1", + "asset": "AAA", + "action": "TARGET", + "strength": 0.5, + "size": 50.0, + }, + { + "time": "t1", + "asset": "BBB", + "action": "TARGET", + "strength": -0.5, + "size": -100.0, + }, + { + "time": "t3", + "asset": "AAA", + "action": "TARGET", + "strength": 0.7, + "size": 20.0, + }, + { + "time": "t3", + "asset": "BBB", + "action": "TARGET", + "strength": -0.7, + "size": -40.0, + }, + ] + assert execution_rows.iloc[4].to_dict() | {"strength": None} == { + "time": "t4", + "asset": "AAA", + "action": "CLOSE", + "strength": None, + "size": -70.0, + } + assert execution_rows.iloc[5].to_dict() | {"strength": None} == { + "time": "t4", + "asset": "BBB", + "action": "CLOSE", + "strength": None, + "size": 140.0, + } + + +def test_calculate_pair_theo_executions_trades_threshold_boundary_and_zero_crossing(): + trd_inst_df = pd.DataFrame( + { + "time_ns": [1, 2, 3, 4], + "tstamp": ["t1", "t2", "t3", "t4"], + "data": [ + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.4"},' + '"BBB":{"reference_price":"50","strength":"0"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.5"},' + '"BBB":{"reference_price":"50","strength":"0"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.51"},' + '"BBB":{"reference_price":"50","strength":"0.1"}}}' + ), + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100"},' + '"BBB":{"reference_price":"50"}}}' + ), + ], + } + ) + + executions = calculate_pair_theo_executions( + "AAA:USD-BBB:USD", + trd_inst_df, + min_pctg_change=25, + ) + + assert executions[["time", "asset", "action", "strength", "size"]].iloc[ + :3 + ].to_dict("records") == [ + { + "time": "t1", + "asset": "AAA", + "action": "TARGET", + "strength": 0.4, + "size": 40.0, + }, + { + "time": "t2", + "asset": "AAA", + "action": "TARGET", + "strength": 0.5, + "size": 10.0, + }, + { + "time": "t3", + "asset": "BBB", + "action": "TARGET", + "strength": 0.1, + "size": 20.0, + }, + ] + + +def test_calculate_pair_theo_ret_uses_execution_cash_flows(): + trd_inst_df = pd.DataFrame( + { + "time_ns": [1, 2, 3], + "data": [ + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.01"},' + '"BBB":{"reference_price":"50","strength":"-0.02"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"110","strength":"0.015"},' + '"BBB":{"reference_price":"45","strength":"-0.01"}}}' + ), + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"120"},' + '"BBB":{"reference_price":"40"}}}' + ), + ], + } + ) + + assert calculate_pair_theo_ret("AAA:USD-BBB:USD", trd_inst_df) == { + "pair_name": "AAA:USD-BBB:USD", + "num_trades": 6, + "realized_pnl": pytest.approx(0.5474747474747474), "unrealized_pnl": 0.0, } @@ -144,6 +764,7 @@ def test_calculate_pair_theo_ret_ignores_unmatched_quote_and_close_without_targe assert calculate_pair_theo_ret("AAA:USD-BBB:USD", trd_inst_df) == { "pair_name": "AAA:USD-BBB:USD", + "num_trades": 0, "realized_pnl": 0.0, "unrealized_pnl": 0.0, } @@ -180,20 +801,297 @@ def test_calculate_ranked_pairs_theo_ret_preserves_pairs_without_instructions(): { "pair_name": "AAA:USD-BBB:USD", "mr_ranking": 1, - "realized_pnl": 20.0, + "num_trades": 4, + "realized_pnl": 0.3, "unrealized_pnl": 0.0, }, { "pair_name": "CCC:USD-DDD:USD", "mr_ranking": 2, + "num_trades": 0, "realized_pnl": 0.0, "unrealized_pnl": 0.0, }, ] +def test_calculate_ranked_pairs_theo_ret_applies_min_pctg_change(): + rankings = pd.DataFrame( + { + "pair_name": ["AAA:USD-BBB:USD"], + "pair_rank": pd.Series([1], dtype="Int64"), + } + ) + trd_inst_df = pd.DataFrame( + { + "time_ns": [1, 2, 3], + "data": [ + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.5"},' + '"BBB":{"reference_price":"50","strength":"-0.5"}}}' + ), + ( + '{"action":"TARGET","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100","strength":"0.53"},' + '"BBB":{"reference_price":"50","strength":"-0.47"}}}' + ), + ( + '{"action":"CLOSE","quote_asset":"USD","assets":' + '{"AAA":{"reference_price":"100"},' + '"BBB":{"reference_price":"50"}}}' + ), + ], + } + ) + + result = calculate_ranked_pairs_theo_ret( + rankings, + trd_inst_df, + min_pctg_change=25, + ) + + assert result.to_dict("records") == [ + { + "pair_name": "AAA:USD-BBB:USD", + "mr_ranking": 1, + "num_trades": 4, + "realized_pnl": 0.0, + "unrealized_pnl": 0.0, + } + ] + + +def test_calculate_ranked_pairs_theo_ret_validates_min_pctg_change(): + rankings = pd.DataFrame( + { + "pair_name": ["AAA:USD-BBB:USD"], + "pair_rank": pd.Series([1], dtype="Int64"), + } + ) + trd_inst_df = pd.DataFrame({"time_ns": [], "data": []}) + + with pytest.raises(ValueError, match="min_pctg_change must be non-negative"): + calculate_ranked_pairs_theo_ret( + rankings, + trd_inst_df, + min_pctg_change=-1, + ) + + def test_load_trading_instructions_validates_required_table(): conn = sqlite3.connect(":memory:") with pytest.raises(ValueError, match="missing required table: trading_instructions"): load_trading_instructions(conn) + + +def test_find_repo_root_and_normalize_directory(): + repo_root = find_repo_root(Path("notebooks").resolve()) + + assert repo_root.name == "stat_pairs_backtest" + assert normalize_directory("data", repo_root) == (repo_root / "data").resolve() + + +def test_list_candidate_files_prefers_result_databases(tmp_path): + names = [ + "20260617.spbt_md.db", + "20260617.spbt_results.db", + "20260617.spbt_selector_results.db", + "notes.txt", + ] + for name in names: + (tmp_path / name).write_text("", encoding="utf-8") + + assert [path.name for path in list_candidate_files(tmp_path)] == [ + "20260617.spbt_results.db", + "20260617.spbt_selector_results.db", + "20260617.spbt_md.db", + ] + assert [path.name for path in list_candidate_files(tmp_path, show_all=True)] == [ + "20260617.spbt_results.db", + "20260617.spbt_selector_results.db", + "20260617.spbt_md.db", + "notes.txt", + ] + + +def test_connect_sqlite_read_only_uses_read_only_uri(tmp_path): + db_path = tmp_path / "example name.sqlite" + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE sample (value INTEGER)") + conn.execute("INSERT INTO sample (value) VALUES (1)") + conn.commit() + conn.close() + + assert read_only_sqlite_uri(db_path).endswith("?mode=ro") + + read_only_conn = connect_sqlite_read_only(db_path) + try: + assert read_only_conn.execute("SELECT value FROM sample").fetchone() == (1,) + with pytest.raises(sqlite3.OperationalError, match="readonly"): + read_only_conn.execute("INSERT INTO sample (value) VALUES (2)") + finally: + read_only_conn.close() + + +def test_add_total_pnl_and_histogram_builder(): + pair_theo_ret = pd.DataFrame( + { + "pair_name": ["AAA:USD-BBB:USD", "PAIR_B"], + "mr_ranking": [1, 2], + "realized_pnl": [1.5, -0.5], + "unrealized_pnl": [0.25, 0.0], + } + ) + + with_total = add_total_pnl(pair_theo_ret) + histogram = create_total_pnl_histogram(pair_theo_ret) + + assert with_total["total_pnl"].tolist() == [1.75, -0.5] + assert histogram.data[0].type == "histogram" + assert histogram.data[0].x.tolist() == [1.75, -0.5] + assert histogram.data[0].xbins.start is None + assert histogram.data[0].xbins.size is None + + +def test_format_pair_names_for_display_removes_usd_suffix_without_mutating_source(): + pair_theo_ret = pd.DataFrame( + { + "pair_name": [ + "AAA:USD-BBB:USD", + "CCC:EUR-DDD:EUR", + "AAA:USDT-BBB:USDT", + None, + ], + "realized_pnl": [1.0, 2.0, 3.0, 4.0], + } + ) + + formatted = format_pair_names_for_display(pair_theo_ret) + + assert format_pair_name_for_display("AAA:USD-BBB:USD") == "AAA-BBB" + assert formatted["pair_name"].tolist() == [ + "AAA-BBB", + "CCC:EUR-DDD:EUR", + "AAA:USDT-BBB:USDT", + None, + ] + assert pair_theo_ret["pair_name"].tolist() == [ + "AAA:USD-BBB:USD", + "CCC:EUR-DDD:EUR", + "AAA:USDT-BBB:USDT", + None, + ] + + +def test_show_interactive_dataframe_uses_sortable_grid_defaults(monkeypatch): + calls = [] + + def fake_show(dataframe, **kwargs): + calls.append((dataframe, kwargs)) + + import itables + + monkeypatch.setattr(itables, "show", fake_show) + dataframe = pd.DataFrame({"pair_name": ["AAA-BBB"], "num_trades": [2]}) + + show_interactive_dataframe( + dataframe, + table_id="pair-theo-ret-grid", + pageLength=50, + ) + + assert len(calls) == 1 + assert calls[0][0] is dataframe + assert calls[0][1] == { + "paging": True, + "pageLength": 50, + "scrollX": True, + "ordering": True, + "showIndex": False, + "maxBytes": "8MB", + "classes": "display compact stripe hover", + "css": calls[0][1]["css"], + "table_id": "pair-theo-ret-grid", + } + assert "background-color: #ffffff" in calls[0][1]["css"] + assert "color: #000000" in calls[0][1]["css"] + + +def test_sorted_pair_names_and_dropdown_use_alphabetical_unique_pairs(): + selector_pair_rankings = pd.DataFrame( + { + "pair_name": [ + "BTC:USD-ETH:USD", + "ADA:USD-BTC:USD", + "BTC:USD-ETH:USD", + ] + } + ) + + assert sorted_pair_names(selector_pair_rankings) == [ + "ADA:USD-BTC:USD", + "BTC:USD-ETH:USD", + ] + + dropdown = create_pair_name_dropdown(selector_pair_rankings) + + assert dropdown.options == ( + ("ADA-BTC", "ADA:USD-BTC:USD"), + ("BTC-ETH", "BTC:USD-ETH:USD"), + ) + assert dropdown.value == "ADA:USD-BTC:USD" + + +def test_example_database_pair_theo_executions_include_strength(): + db_path = Path("data/20260617.spbt_results.db") + if not db_path.exists(): + pytest.skip(f"example database not available: {db_path}") + + conn = connect_sqlite_read_only(db_path) + try: + rankings = load_selector_pair_rankings(conn) + trading_instructions = load_trading_instructions(conn) + finally: + conn.close() + + empty_pair_executions = calculate_pair_theo_executions( + "ADA:USD-BNB:USD", + trading_instructions, + ) + non_empty_pair_executions = calculate_pair_theo_executions( + "ADA:USD-BTC:USD", + trading_instructions, + ) + + assert "strength" in empty_pair_executions.columns + assert "strength" in non_empty_pair_executions.columns + assert len(rankings) == 78 + assert len(non_empty_pair_executions) > 0 + assert ( + non_empty_pair_executions.loc[ + non_empty_pair_executions["action"] == "TARGET", + "strength", + ] + .notna() + .all() + ) + assert ( + non_empty_pair_executions.loc[ + non_empty_pair_executions["action"] == "CLOSE", + "strength", + ] + .isna() + .all() + ) + + first_target_execution = non_empty_pair_executions[ + non_empty_pair_executions["action"] == "TARGET" + ].iloc[0] + assert first_target_execution["size"] == pytest.approx( + 10_000 * first_target_execution["strength"] / first_target_execution["price"] + ) + assert first_target_execution["usd_value"] == pytest.approx( + -first_target_execution["size"] * first_target_execution["price"] + )