{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import plotly.figure_factory as ff\n", "import pandas as pd\n", "import pickle\n", "import plotly.graph_objs as go\n", "import latextable\n", "from texttable import Texttable\n", "from strategy.strategy import (\n", " BuyAndHoldStrategy,\n", " MACDStrategy,\n", " RSIStrategy,\n", " ModelQuantilePredictionsStrategy,\n", " ModelGmadlPredictionsStrategy,\n", " ConcatenatedStrategies\n", ")\n", "from strategy.util import (\n", " get_data_windows,\n", " get_sweep_window_predictions,\n", " get_predictions_dataframe\n", ")\n", "from strategy.evaluation import (\n", " parameter_sweep,\n", " evaluate_strategy\n", ")\n", "from strategy.plotting import (\n", " plot_sweep_results\n", ")\n", "\n", "PADDING=5000\n", "VALID_PART=0.2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-5m:latest, 745.12MB. 12 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 12 of 12 files downloaded. \n", "Done. 0:0:1.1\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-15m:latest, 248.65MB. 12 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 12 of 12 files downloaded. \n", "Done. 0:0:0.5\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-30m:latest, 124.19MB. 12 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 12 of 12 files downloaded. \n", "Done. 0:0:0.4\n" ] } ], "source": [ "data_windows_5min = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-5m:latest',\n", " min_window=0, \n", " max_window=5\n", ")\n", "\n", "data_windows_15min = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-15m:latest',\n", " min_window=0, \n", " max_window=5\n", ")\n", "\n", "data_windows_30min = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-30m:latest',\n", " min_window=0, \n", " max_window=5\n", ")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "with open('cache/5min-best-strategies-v2.pkl', 'rb') as inpt:\n", " best_strategies_5min = pickle.load(inpt)\n", "\n", "with open('cache/15min-best-strategies-v2.pkl', 'rb') as inpt:\n", " best_strategies_15min = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies-v6.pkl', 'rb') as inpt:\n", " best_strategies_30min = pickle.load(inpt)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Best 5 strategies evaluation" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "def results_plot(\n", " buy_and_hold_concat,\n", " macd_30min_concat,\n", " rsi_30min_concat,\n", " rsi_5_min_concat,\n", " rmse_30min_model_concat,\n", " rmse_15min_model_concat,\n", " gmadl_30min_model_concat,\n", " gmadl_15min_model_concat,\n", " gmadl_5min_model_concat, width=850, height=500, notitle=False):\n", "\n", " fig = go.Figure([\n", " go.Scatter(y=buy_and_hold_concat['portfolio_value'], x=buy_and_hold_concat['time'], name=\"Buy and Hold\"),\n", " go.Scatter(y=macd_30min_concat['portfolio_value'], x=macd_30min_concat['time'], name='MACD Strategy (30min)'),\n", " go.Scatter(y=rsi_30min_concat['portfolio_value'], x=rsi_30min_concat['time'], name='RSI Strategy (30min)'),\n", " go.Scatter(y=rsi_5_min_concat['portfolio_value'], x=rsi_5_min_concat['time'], name=\"RSI Strategy (5min)\"),\n", " go.Scatter(y=rmse_30min_model_concat['portfolio_value'], x=rmse_30min_model_concat['time'], name='RMSE Informer Strategy (30min)'),\n", " go.Scatter(y=rmse_15min_model_concat['portfolio_value'], x=rmse_15min_model_concat['time'], name='RMSE Informer Strategy (15min)'),\n", " go.Scatter(y=gmadl_30min_model_concat['portfolio_value'], x=gmadl_30min_model_concat['time'], name='GMADL Informer Strategy (30min)'),\n", " go.Scatter(y=gmadl_15min_model_concat['portfolio_value'], x=gmadl_15min_model_concat['time'], name='GMADL Informer Strategy (15min)'),\n", " go.Scatter(y=gmadl_5min_model_concat['portfolio_value'], x=gmadl_5min_model_concat['time'], name=\"GMADL Informer Strategy (5min)\")\n", " ])\n", " fig.update_layout(\n", " # title={\n", " # 'text': f\"W{idx}-{INTERVAL}\",\n", " # 'y':0.97,\n", " # 'x':0.5,\n", " # 'xanchor': 'center',\n", " # 'yanchor': 'top'} if not notitle else None,\n", " yaxis_title=\"Portfolio Value\",\n", " xaxis_title=\"Date\",\n", " font=dict(\n", " # family=\"Courier New, monospace\",\n", " size=14,\n", " ),\n", " autosize=False,\n", " width=width,\n", " height=height,\n", " margin=dict(l=20, r=20, t=50, b=20),\n", " plot_bgcolor='white',\n", " legend=dict(\n", " orientation=\"h\",\n", " yanchor=\"bottom\",\n", " y=1.02,\n", " xanchor=\"left\",\n", " x=0.02\n", " )\n", " )\n", " fig.update_xaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " fig.update_yaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " # fig.write_image(f\"images/eval-w{idx}-{INTERVAL}.png\")\n", " fig.show()\n", " \n", "def results_table(\n", " buy_and_hold_concat,\n", " macd_30min_concat,\n", " rsi_30min_concat,\n", " rsi_5_min_concat,\n", " rmse_30min_model_concat,\n", " rmse_15min_model_concat,\n", " gmadl_30min_model_concat,\n", " gmadl_15min_model_concat,\n", " gmadl_5min_model_concat):\n", "\n", " table_eval_windows = Texttable()\n", " table_eval_windows.set_deco(Texttable.HEADER)\n", " table_eval_windows.set_cols_align([\"l\", \"c\",\"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\"])\n", " table_eval_windows.set_precision(3)\n", "\n", " table_eval_windows.header([\n", " \"\\\\textbf{Strategy}\",\n", " \"\\\\textbf{VAL}\",\n", " \"\\\\textbf{ARC}\",\n", " \"\\\\textbf{ASD}\",\n", " \"\\\\textbf{IR*}\",\n", " \"\\\\textbf{MD}\",\n", " \"\\\\textbf{IR**}\",\n", " \"\\\\textbf{N}\",\n", " \"\\\\textbf{LONG}\",\n", " \"\\\\textbf{SHORT}\",\n", " ])\n", "\n", " strategy_name_result = [\n", " ('Buy and Hold', buy_and_hold_concat),\n", " ('MACD Strategy (30min)', macd_30min_concat),\n", " ('RSI Strategy (30min)', rsi_30min_concat),\n", " ('RSI Strategy (5min)', rsi_5_min_concat),\n", " ('RMSE Informer (30min)', rmse_30min_model_concat),\n", " ('RMSE Informer (15min)', rmse_15min_model_concat),\n", " ('GMADL Informer (30min)', gmadl_30min_model_concat),\n", " ('GMADL Informer (15min)', gmadl_15min_model_concat),\n", " ('GMADL Informer (5min)', gmadl_5min_model_concat),\n", " ]\n", " for strategy_name, result in strategy_name_result:\n", " table_eval_windows.add_row([\n", " strategy_name,\n", " result['value'],\n", " result['arc'],\n", " result['asd'],\n", " result['ir'],\n", " result['md'],\n", " result['mod_ir'],\n", " result['n_trades'],\n", " f\"{result['long_pos']*100:.2f}\\%\",\n", " f\"{result['short_pos']*100:.2f}\\%\",\n", " ])\n", " print(latextable.draw_latex(table_eval_windows))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 1.441 & 0.131 & 0.577 & 0.228 & 0.773 & 0.039 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tMACD Strategy (30min) & 1.952 & 0.254 & 0.524 & 0.485 & 0.592 & 0.207 & 327 & 52.30\\% & 28.30\\% \\\\\n", "\t\t\tRSI Strategy (30min) & 4.542 & 0.668 & 0.462 & 1.444 & 0.399 & 2.415 & 377 & 30.79\\% & 28.03\\% \\\\\n", "\t\t\tRSI Strategy (5min) & 3.341 & 0.503 & 0.504 & 0.999 & 0.300 & 1.676 & 846 & 28.29\\% & 33.47\\% \\\\\n", "\t\t\tRMSE Informer (30min) & 2.727 & 0.404 & 0.505 & 0.800 & 0.518 & 0.624 & 34 & 64.40\\% & 24.67\\% \\\\\n", "\t\t\tRMSE Informer (15min) & 1.509 & 0.149 & 0.349 & 0.428 & 0.455 & 0.140 & 16 & 15.24\\% & 27.60\\% \\\\\n", "\t\t\tGMADL Informer (30min) & 2.263 & 0.318 & 0.367 & 0.866 & 0.533 & 0.516 & 811 & 35.51\\% & 19.59\\% \\\\\n", "\t\t\tGMADL Informer (15min) & 3.296 & 0.496 & 0.527 & 0.942 & 0.474 & 0.987 & 362 & 49.37\\% & 37.72\\% \\\\\n", "\t\t\tGMADL Informer (5min) & 9.747 & 1.159 & 0.544 & 2.129 & 0.327 & 7.552 & 846 & 44.80\\% & 41.51\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n" ] } ], "source": [ "test_data_5min = pd.concat([data_windows_5min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_5min])\n", "test_data_15min = pd.concat([data_windows_15min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_15min])\n", "test_data_30min = pd.concat([data_windows_30min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_30min])\n", "\n", "buy_and_hold_concat = evaluate_strategy(test_data_5min, BuyAndHoldStrategy(), padding=PADDING, interval='5min')\n", "macd_30min_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['macd_strategies']], padding=PADDING), padding=PADDING, interval='30min')\n", "rsi_30min_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='30min')\n", "rsi_5_min_concat = evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[0] for s in best_strategies_5min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='5min')\n", "rmse_30min_model_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['rmse_model']], padding=PADDING), padding=PADDING, interval='30min')\n", "rmse_15min_model_concat = evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[0] for s in best_strategies_15min['rmse_model']], padding=PADDING), padding=PADDING, interval='15min')\n", "gmadl_30min_model_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['gmadl_model']], padding=PADDING), padding=PADDING, interval='30min')\n", "gmadl_15min_model_concat = evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[0] for s in best_strategies_15min['gmadl_model']], padding=PADDING), padding=PADDING, interval='15min')\n", "gmadl_5min_model_concat = evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[0] for s in best_strategies_5min['gmadl_model']], padding=PADDING), padding=PADDING, interval='5min')\n", "\n", "results_table(\n", " buy_and_hold_concat,\n", " macd_30min_concat,\n", " rsi_30min_concat,\n", " rsi_5_min_concat,\n", " rmse_30min_model_concat,\n", " rmse_15min_model_concat,\n", " gmadl_30min_model_concat,\n", " gmadl_15min_model_concat,\n", " gmadl_5min_model_concat)\n", "\n", "# results_plot(\n", "# buy_and_hold_concat,\n", "# macd_30min_concat,\n", "# rsi_30min_concat,\n", "# rsi_5_min_concat,\n", "# rmse_30min_model_concat,\n", "# rmse_15min_model_concat,\n", "# gmadl_30min_model_concat,\n", "# gmadl_15min_model_concat,\n", "# gmadl_5min_model_concat, \n", "# width=1200, notitle=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Statistical significance" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "test_data_5min = pd.concat([data_windows_5min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_5min])\n", "test_data_15min = pd.concat([data_windows_15min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_15min])\n", "test_data_30min = pd.concat([data_windows_30min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_30min])\n", "\n", "buy_and_hold_5min = evaluate_strategy(test_data_5min, BuyAndHoldStrategy(), padding=PADDING, interval='5min')\n", "buy_and_hold_15min = evaluate_strategy(test_data_15min, BuyAndHoldStrategy(), padding=PADDING, interval='15min')\n", "buy_and_hold_30min = evaluate_strategy(test_data_30min, BuyAndHoldStrategy(), padding=PADDING, interval='30min')\n", "macd_30min_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['macd_strategies']], padding=PADDING), padding=PADDING, interval='30min')\n", "rsi_30min_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='30min')\n", "rsi_5_min_concat = evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[0] for s in best_strategies_5min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='5min')\n", "rmse_30min_model_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['rmse_model']], padding=PADDING), padding=PADDING, interval='30min')\n", "rmse_15min_model_concat = evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[0] for s in best_strategies_15min['rmse_model']], padding=PADDING), padding=PADDING, interval='15min')\n", "gmadl_30min_model_concat = evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[0] for s in best_strategies_30min['gmadl_model']], padding=PADDING), padding=PADDING, interval='30min')\n", "gmadl_15min_model_concat = evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[0] for s in best_strategies_15min['gmadl_model']], padding=PADDING), padding=PADDING, interval='15min')\n", "gmadl_5min_model_concat = evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[0] for s in best_strategies_5min['gmadl_model']], padding=PADDING), padding=PADDING, interval='5min')" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "MACD (30 min) & 51840 & 0.326504 & 174.34 & 0.000000*** \\\\\n", "RSI (30 min) & 51840 & 1.065079 & 258.49 & 0.000000*** \\\\\n", "RSI (5 min) & 311040 & 0.648741 & 662.77 & 0.000000*** \\\\\n", "RMSE (30 min) & 51840 & 0.796647 & 161.58 & 0.000000*** \\\\\n", "RMSE (15 min) & 103680 & 0.541611 & 115.29 & 0.000000*** \\\\\n", "GMADL (30 min) & 51840 & 0.320689 & 448.49 & 0.000000*** \\\\\n", "GMADL (15 min) & 103680 & 0.558743 & 408.13 & 0.000000*** \\\\\n", "GMADL (5min) & 311040 & 2.820834 & 375.84 & 0.000000*** \\\\\n" ] } ], "source": [ "import numpy as np\n", "import scipy.stats\n", "\n", "def ttest(benchmark, strategy):\n", " sigma = np.std(strategy['portfolio_value'] - benchmark['portfolio_value'])\n", " N = len(strategy['strategy_returns'])\n", " tt = (strategy['ir'] - benchmark['ir']) / (sigma / np.sqrt(N))\n", "\n", " pval = scipy.stats.t.sf(np.abs(tt), N-1)*2\n", "\n", " return tt, pval, sigma, N\n", "\n", "macd_30min_ttest = ttest(buy_and_hold_30min, macd_30min_concat)\n", "rsi_30min_ttest = ttest(buy_and_hold_30min, rsi_30min_concat)\n", "rsi_5min_ttest = ttest(buy_and_hold_5min, rsi_5_min_concat)\n", "rmse_30min_ttest = ttest(buy_and_hold_30min, rmse_30min_model_concat)\n", "rmse_15min_ttest = ttest(buy_and_hold_15min, rmse_15min_model_concat)\n", "gmadl_30min_ttest = ttest(buy_and_hold_30min, gmadl_30min_model_concat)\n", "gmadl_15min_ttest = ttest(buy_and_hold_15min, gmadl_15min_model_concat)\n", "gmadl_5min_ttest = ttest(buy_and_hold_5min, gmadl_5min_model_concat)\n", "\n", "for name, result in [\n", " (\"MACD (30 min)\", macd_30min_ttest),\n", " (\"RSI (30 min)\", rsi_30min_ttest),\n", " (\"RSI (5 min)\", rsi_5min_ttest),\n", " (\"RMSE (30 min)\", rmse_30min_ttest),\n", " (\"RMSE (15 min)\", rmse_15min_ttest),\n", " (\"GMADL (30 min)\", gmadl_30min_ttest),\n", " (\"GMADL (15 min)\", gmadl_15min_ttest),\n", " (\"GMADL (5min)\", gmadl_5min_ttest),\n", "]:\n", " print(f\"{name} & {result[3]} & {result[2]:.6f} & {result[0]:.2f} & {result[1]:.6f}*** \\\\\\\\\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## TOP N Strategies Sensistivity Analisys" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def results_for_strats(\n", " data_windows_5min, data_windows_15min, data_windows_30min, \n", " best_strategies_5min, best_strategies_15min, best_strategies_30min,\n", " top_n=10):\n", " test_data_5min = pd.concat([data_windows_5min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_5min])\n", " test_data_15min = pd.concat([data_windows_15min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_15min])\n", " test_data_30min = pd.concat([data_windows_30min[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows_30min])\n", "\n", " buy_and_hold_concat = evaluate_strategy(test_data_5min, BuyAndHoldStrategy(), padding=PADDING, interval='5min')\n", " macd_30min_concat = [evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[x] for s in best_strategies_30min['macd_strategies']], padding=PADDING), padding=PADDING, interval='30min') for x in range(top_n)]\n", " macd_15min_concat = [evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[x] for s in best_strategies_15min['macd_strategies']], padding=PADDING), padding=PADDING, interval='15min') for x in range(top_n)]\n", " macd_5min_concat = [evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[x] for s in best_strategies_5min['macd_strategies']], padding=PADDING), padding=PADDING, interval='5min') for x in range(top_n)]\n", " rsi_30min_concat = [evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[x] for s in best_strategies_30min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='30min') for x in range(top_n)]\n", " rsi_15min_concat = [evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[x] for s in best_strategies_15min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='15min') for x in range(top_n)]\n", " rsi_5_min_concat = [evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[x] for s in best_strategies_5min['rsi_strategies']], padding=PADDING), padding=PADDING, interval='5min') for x in range(top_n)]\n", " rmse_30min_model_concat = [evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[x] for s in best_strategies_30min['rmse_model']], padding=PADDING), padding=PADDING, interval='30min') for x in range(top_n)]\n", " rmse_15min_model_concat = [evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[x] for s in best_strategies_15min['rmse_model']], padding=PADDING), padding=PADDING, interval='15min') for x in range(top_n)]\n", " rmse_5min_model_concat = [evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[x] for s in best_strategies_5min['rmse_model']], padding=PADDING), padding=PADDING, interval='5min') for x in range(top_n)]\n", " quantile_30min_model_concat = [evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[x] for s in best_strategies_30min['quantile_model']], padding=PADDING), padding=PADDING, interval='30min') for x in range(top_n)]\n", " quantile_15min_model_concat = [evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[x] for s in best_strategies_15min['quantile_model']], padding=PADDING), padding=PADDING, interval='15min') for x in range(top_n)]\n", " quantile_5min_model_concat = [evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[x] for s in best_strategies_5min['quantile_model']], padding=PADDING), padding=PADDING, interval='5min') for x in range(top_n)]\n", " gmadl_30min_model_concat = [evaluate_strategy(test_data_30min, ConcatenatedStrategies(len(data_windows_30min[0][1]), [s[x] for s in best_strategies_30min['gmadl_model']], padding=PADDING), padding=PADDING, interval='30min') for x in range(top_n)]\n", " gmadl_15min_model_concat = [evaluate_strategy(test_data_15min, ConcatenatedStrategies(len(data_windows_15min[0][1]), [s[x] for s in best_strategies_15min['gmadl_model']], padding=PADDING), padding=PADDING, interval='15min') for x in range(top_n)]\n", " gmadl_5min_model_concat = [evaluate_strategy(test_data_5min, ConcatenatedStrategies(len(data_windows_5min[0][1]), [s[x] for s in best_strategies_5min['gmadl_model']], padding=PADDING), padding=PADDING, interval='5min') for x in range(top_n)]\n", "\n", " z = list(reversed([\n", " list(reversed([round(buy_and_hold_concat['mod_ir'], 3)]*top_n)),\n", " list(reversed([round(x['mod_ir'], 3) for x in macd_30min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in macd_15min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in macd_5min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rsi_30min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rsi_15min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rsi_5_min_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rmse_30min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rmse_15min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in rmse_5min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in quantile_30min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in quantile_15min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in quantile_5min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in gmadl_30min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in gmadl_15min_model_concat])),\n", " list(reversed([round(x['mod_ir'], 3) for x in gmadl_5min_model_concat])),\n", " ]))\n", " x = list(reversed(range(1, top_n+1)))\n", " y = list(reversed([\n", " \"Buy and Hold\",\n", " \"MACD Strategy (30 min)\",\n", " \"MACD Strategy (15 min)\",\n", " \"MACD Strategy (5 min)\",\n", " \"RSI Strategy (30 min)\",\n", " \"RSI Strategy (15 min)\",\n", " \"RSI Strategy (5 min)\",\n", " \"RMSE Informer (30 min)\",\n", " \"RMSE Informer (15 min)\",\n", " \"RMSE Informer (5 min)\",\n", " \"Quantile Informer (30 min)\",\n", " \"Quantile Informer (15 min)\",\n", " \"Quantile Informer (5 min)\",\n", " \"Gmadl Informer (30 min)\",\n", " \"Gmadl Informer (15 min)\",\n", " \"Gmadl Informer (5 min)\"\n", " ]))\n", " # 'Portland'\n", " fig = ff.create_annotated_heatmap(z, x=x, y=y, colorscale='thermal', zmid=buy_and_hold_concat['mod_ir'])\n", " fig.update_layout(\n", " margin=dict(l=20, r=20, b=20, t=20),\n", " width=1100,\n", " height=650,\n", " font=dict(\n", " # family=\"Courier New, monospace\",\n", " size=16, # Set the font size here\n", " # color=\"RebeccaPurple\"\n", " )\n", " )\n", " fig.show()\n", "\n", "# results_for_strats(data_windows_5min, data_windows_15min, data_windows_30min,\n", "# best_strategies_5min, best_strategies_15min, best_strategies_30min, top_n=10) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Validation size for: GMADL Informer (5min)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "with open('cache/5min-best-strategies-01.pkl', 'rb') as inpt:\n", " best_strategies_5min_01 = pickle.load(inpt)\n", "\n", "with open('cache/5min-best-strategies.pkl', 'rb') as inpt:\n", " best_strategies_5min_02 = pickle.load(inpt)\n", "\n", "with open('cache/5min-best-strategies-03.pkl', 'rb') as inpt:\n", " best_strategies_5min_03 = pickle.load(inpt)\n", "\n", "with open('cache/5min-best-strategies-04.pkl', 'rb') as inpt:\n", " best_strategies_5min_04 = pickle.load(inpt)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'strategy_name': 'Unknown model', 'future': 1, 'target': 'close_price', 'enter_long': 0.004, 'exit_long': None, 'enter_short': -0.005, 'exit_short': None}\n", "{'strategy_name': 'Unknown model', 'future': 1, 'target': 'close_price', 'enter_long': 0.007, 'exit_long': -0.003, 'enter_short': None, 'exit_short': None}\n" ] } ], "source": [ "# print(best_strategies_5min_02['gmadl_model'][0][0].info())\n", "# print(best_strategies_5min_03['gmadl_model'][0][0].info())" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [], "source": [ "def results_plot2(result_buyandhold, results, names, width=850, height=500):\n", "\n", " fig = go.Figure([\n", " go.Scatter(y=result_buyandhold['portfolio_value'], x=result_buyandhold['time'], name=\"Buy and Hold\")] +[\n", " go.Scatter(y=result['portfolio_value'], x=result['time'], name=name) for name, result in zip(names, results)\n", " ])\n", " # go.Scatter(y=result_macd['portfolio_value'], x=result_macd['time'], name=\"MACD Strategy\"),\n", " # go.Scatter(y=result_rsi['portfolio_value'], x=result_rsi['time'], name=\"RSI Strategy\"),\n", " # go.Scatter(y=result_quantile_model['portfolio_value'], x=result_quantile_model['time'], name='Quantile Informer Strategy'),\n", " # go.Scatter(y=result_gmadl_model['portfolio_value'], x=result_gmadl_model['time'], name='GMADL Informer Strategy')\n", " # ])\n", " fig.update_layout(\n", " yaxis_title=\"Portfolio Value\",\n", " xaxis_title=\"Date\",\n", " font=dict(\n", " # family=\"Courier New, monospace\",\n", " size=14,\n", " ),\n", " autosize=False,\n", " width=width,\n", " height=height,\n", " margin=dict(l=20, r=20, t=20, b=20),\n", " plot_bgcolor='white',\n", " legend=dict(\n", " orientation=\"h\",\n", " yanchor=\"bottom\",\n", " y=1.02,\n", " xanchor=\"left\",\n", " x=0.02\n", " )\n", " )\n", " fig.update_xaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " fig.update_yaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " fig.show()\n", " \n", "def results_table2(result_buyandhold, results, names):\n", " table_eval_windows = Texttable()\n", " table_eval_windows.set_deco(Texttable.HEADER)\n", " table_eval_windows.set_cols_align([\"l\", \"c\",\"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\"])\n", " table_eval_windows.set_precision(2)\n", "\n", " table_eval_windows.header([\n", " \"\\\\textbf{Strategy}\",\n", " \"\\\\textbf{VAL}\",\n", " \"\\\\textbf{ARC}\",\n", " \"\\\\textbf{ASD}\",\n", " \"\\\\textbf{IR*}\",\n", " \"\\\\textbf{MD}\",\n", " \"\\\\textbf{IR**}\",\n", " \"\\\\textbf{N}\",\n", " \"\\\\textbf{LONG}\",\n", " \"\\\\textbf{SHORT}\",\n", " ])\n", "\n", " strategy_name_result = [\n", " ('Buy and Hold', result_buyandhold)] + list(zip(names, results))\n", " for strategy_name, result in strategy_name_result:\n", " table_eval_windows.add_row([\n", " strategy_name,\n", " result['value'],\n", " f\"{result['arc']*100:.1f}\\%\",\n", " f\"{result['asd']*100:.1f}\\%\",\n", " result['ir'],\n", " f\"{result['md']*100:.1f}\\%\",\n", " result['mod_ir'],\n", " result['n_trades'],\n", " f\"{result['long_pos']*100:.1f}\\%\",\n", " f\"{result['short_pos']*100:.1f}\\%\",\n", " ])\n", " print(latextable.draw_latex(table_eval_windows))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": [ "def results_for_strats(data_windows, best_strategies, names, interval='5min', strat_name='gmadl_model'):\n", " test_data = pd.concat([data_windows[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows])\n", "\n", " buy_and_hold_concat = evaluate_strategy(test_data, BuyAndHoldStrategy(), padding=PADDING, interval=interval)\n", " \n", " evaluation_results = []\n", " for best_strat in best_strategies:\n", " evaluation_results.append(\n", " evaluate_strategy(test_data, ConcatenatedStrategies(len(data_windows[0][1]), [s[0] for s in best_strat[strat_name]], padding=PADDING), padding=PADDING, interval=interval)\n", " )\n", " results_plot(buy_and_hold_concat, evaluation_results, names, width=1200)\n", " results_table2(buy_and_hold_concat, evaluation_results, names)\n", "\n", "\n", "# results_for_strats(data_windows_5min, [\n", "# best_strategies_5min_01,\n", "# best_strategies_5min_02,\n", "# best_strategies_5min_03,\n", "# best_strategies_5min_04\n", "# ], [\n", "# \"3 months\",\n", "# \"6 months\",\n", "# \"9 months\",\n", "# \"12 months\",\n", "# ])\n" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [], "source": [ "with open('cache/30min-best-strategies-01.pkl', 'rb') as inpt:\n", " best_strategies_30min_01 = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies.pkl', 'rb') as inpt:\n", " best_strategies_30min_02 = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies-03.pkl', 'rb') as inpt:\n", " best_strategies_30min_03 = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies-04.pkl', 'rb') as inpt:\n", " best_strategies_30min_04 = pickle.load(inpt)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# results_for_strats(data_windows_30min, [\n", "# best_strategies_30min_01,\n", "# best_strategies_30min_02,\n", "# best_strategies_30min_03,\n", "# best_strategies_30min_04\n", "# ], [\n", "# \"3 months\",\n", "# \"6 months\",\n", "# \"9 months\",\n", "# \"12 months\",\n", "# ], strat_name='rsi_strategies', interval='30min')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# test_data = pd.concat([data_window[0] for data_window in data_windows_5min[:1]])\n", "# buy_and_hold_concat = evaluate_strategy(test_data, BuyAndHoldStrategy(), padding=PADDING)\n", "\n", "# res = evaluate_strategy(test_data, ConcatenatedStrategies(1, [s[0] for s in best_strategies_5min_02['gmadl_model'][:1]], padding=PADDING), padding=PADDING, interval='5min')\n", "# results_plot(buy_and_hold_concat, [res], width=1200)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Different number of test windows" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "with open('cache/5min-best-strategies-short.pkl', 'rb') as inpt:\n", " best_strategies_5min_short = pickle.load(inpt)\n", "\n", "with open('cache/5min-best-strategies-long.pkl', 'rb') as inpt:\n", " best_strategies_5min_long = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies-short.pkl', 'rb') as inpt:\n", " best_strategies_30min_short = pickle.load(inpt)\n", "\n", "with open('cache/30min-best-strategies-long.pkl', 'rb') as inpt:\n", " best_strategies_30min_long = pickle.load(inpt)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-5m-short:latest, 1341.36MB. 24 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 24 of 24 files downloaded. \n", "Done. 0:0:0.7\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-5m-long:latest, 446.99MB. 6 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 6 of 6 files downloaded. \n", "Done. 0:0:1.1\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-30m-short:latest, 223.56MB. 24 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 24 of 24 files downloaded. \n", "Done. 0:0:0.7\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-30m-long:latest, 74.51MB. 6 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 6 of 6 files downloaded. \n", "Done. 0:0:0.4\n" ] } ], "source": [ "data_windows_5min_short = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-5m-short:latest',\n", " min_window=0, \n", " max_window=11\n", ")\n", "\n", "data_windows_5min_long = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-5m-long:latest',\n", " min_window=0, \n", " max_window=2\n", ")\n", "\n", "data_windows_30min_short = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-30m-short:latest',\n", " min_window=0, \n", " max_window=11\n", ")\n", "\n", "data_windows_30min_long = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-30m-long:latest',\n", " min_window=0, \n", " max_window=2\n", ")" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [], "source": [ "# def results_for_strats_different_windows(buy_and_hold_data_windows, *args, interval='5min'):\n", "# evaluation_results = []\n", "# names = []\n", "# buy_and_hold_data = pd.concat([buy_and_hold_data_windows[0][0][-PADDING:]] + [data_window[1] for data_window in buy_and_hold_data_windows])\n", "# buy_and_hold_concat = evaluate_strategy(buy_and_hold_data, BuyAndHoldStrategy(), padding=PADDING, interval=interval)\n", "# for data_windows, strategy, name in args:\n", "# test_data = pd.concat([data_windows[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows])\n", "# # buy_and_hold_concat = evaluate_strategy(test_data, BuyAndHoldStrategy(), padding=PADDING, interval=interval)\n", "# evaluation_results.append(\n", "# evaluate_strategy(test_data, ConcatenatedStrategies(len(data_windows[0][1]), [s[0] for s in strategy], padding=PADDING), padding=PADDING, interval=interval\n", "# ))\n", "# names.append(name)\n", " \n", "# results_plot2(buy_and_hold_concat, evaluation_results, names, width=1200)\n", "# results_table2(buy_and_hold_concat, evaluation_results, names)\n", "\n", "# results_for_strats_different_windows(\n", "# data_windows_30min,\n", "# (data_windows_30min_short, best_strategies_30min_short['rsi_strategies'], '12 windows'),\n", "# (data_windows_30min, best_strategies_30min['rsi_strategies'], '6 windows'),\n", "# (data_windows_30min_long, best_strategies_30min_long['rsi_strategies'], '3 windows'),\n", "# interval='30min'\n", "# )\n", "\n", "# results_for_strats_different_windows(\n", "# data_windows_5min,\n", "# (data_windows_5min_short, best_strategies_5min_short['gmadl_model'], '12 windows'),\n", "# (data_windows_5min, best_strategies_5min['gmadl_model'], '6 windows'),\n", "# (data_windows_5min_long, best_strategies_5min_long['gmadl_model'], '3 windows'),\n", "# interval='5min'\n", "# )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "wnemsc", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.19" } }, "nbformat": 4, "nbformat_minor": 2 }