{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import pickle\n", "import plotly.graph_objs as go\n", "import latextable\n", "from texttable import Texttable\n", "from strategy.strategy import (\n", " BuyAndHoldStrategy,\n", " MACDStrategy,\n", " RSIStrategy,\n", " ModelQuantilePredictionsStrategy,\n", " ModelGmadlPredictionsStrategy,\n", " ConcatenatedStrategies\n", ")\n", "from strategy.util import (\n", " get_data_windows,\n", " get_sweep_window_predictions,\n", " get_predictions_dataframe\n", ")\n", "from strategy.evaluation import (\n", " parameter_sweep,\n", " evaluate_strategy\n", ")\n", "from strategy.plotting import (\n", " plot_sweep_results\n", ")\n", "\n", "PADDING=5000\n", "VALID_PART=0.2\n", "INTERVAL='min'\n", "METRIC='mod_ir'\n", "TOP_N=10" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact btc-usdt-1m:latest, 3717.80MB. 12 files... \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 12 of 12 files downloaded. \n", "Done. 0:0:4.7\n" ] } ], "source": [ "data_windows = get_data_windows(\n", " 'wne-masters-thesis-testing',\n", " 'btc-usdt-1m:latest',\n", " min_window=0, \n", " max_window=5\n", ")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def sweeps_on_all_windows(data_windows, strategy_class, params, **kwargs):\n", " result = []\n", " for in_sample, _ in data_windows:\n", " data_part = int((1 - VALID_PART) * len(in_sample))\n", " result.append(parameter_sweep(in_sample[data_part-PADDING:], strategy_class, params, padding=PADDING, interval=INTERVAL, **kwargs))\n", " return result" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "buyandhold_best_strategies = [BuyAndHoldStrategy() for _ in data_windows] " ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n", "\u001b[34m\u001b[1mwandb\u001b[0m: 2 of 2 files downloaded. \n" ] } ], "source": [ "# Model with gmadl loss\n", "SWEEP_ID = 'filipstefaniuk/wne-masters-thesis-testing/s8goxcbz'\n", "# SWEEP_ID = 'filipstefaniuk/wne-masters-thesis-testing/v3epl3qk'\n", "# train_gmadl_pred_windows = get_sweep_window_predictions(SWEEP_ID, 'train')\n", "valid_gmadl_pred_windows = get_sweep_window_predictions(SWEEP_ID, 'valid')\n", "test_gmadl_pred_windows = get_sweep_window_predictions(SWEEP_ID, 'test')" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# y = test_gmadl_pred_windows[0][2][:, 0, 0]\n", "# fig = go.Figure([\n", "# go.Scatter(y=y[::100]),\n", "# ])\n", "# fig.show()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1176/1176 [04:40<00:00, 4.20it/s]\n", "100%|██████████| 1176/1176 [04:40<00:00, 4.20it/s]\n", "100%|██████████| 1176/1176 [04:36<00:00, 4.26it/s]\n", "100%|██████████| 1176/1176 [04:35<00:00, 4.28it/s]\n", "100%|██████████| 1176/1176 [04:36<00:00, 4.26it/s]\n", "100%|██████████| 1176/1176 [04:30<00:00, 4.35it/s]\n" ] } ], "source": [ "MODEL_GMADL_LOSS_FILTER = lambda p: (\n", " ((p['enter_long'] is not None and (p['enter_short'] is not None or p['exit_long'] is not None))\n", " or (p['enter_short'] is not None and (p['exit_short'] is not None or p['enter_long'] is not None)))\n", " and (p['enter_short'] is None or p['exit_long'] is None or (p['exit_long'] > p['enter_short']))\n", " and (p['enter_long'] is None or p['exit_short'] is None or (p['exit_short'] < p['enter_long'])))\n", "\n", "gmadl_model_sweep_results = []\n", "for (in_sample, _), valid_preds, test_preds in zip(data_windows, valid_gmadl_pred_windows, test_gmadl_pred_windows):\n", " data_part = int((1 - VALID_PART) * len(in_sample))\n", " params={\n", " 'predictions': [get_predictions_dataframe(valid_preds, test_preds)],\n", " 'enter_long': [None, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007],\n", " 'exit_long': [None, -0.001, -0.002, -0.003, -0.004, -0.005, -0.006, -0.007],\n", " 'enter_short': [None, -0.001, -0.002, -0.003, -0.004, -0.005, -0.006, -0.007],\n", " 'exit_short': [None, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007],\n", " }\n", " \n", " gmadl_model_sweep_results.append(parameter_sweep(\n", " in_sample[data_part-PADDING:],\n", " ModelGmadlPredictionsStrategy,\n", " params,\n", " params_filter=MODEL_GMADL_LOSS_FILTER,\n", " padding=PADDING,\n", " interval=INTERVAL,\n", " sort_by=METRIC))\n", " \n", "\n", "gmadl_model_best_strategies = [[strat for _, strat in results[:TOP_N]] for results in gmadl_model_sweep_results]" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'buyandhold_best_strategies' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[3], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Persist best strategies, so that they don't have to be recomputed every time\u001b[39;00m\n\u001b[1;32m 2\u001b[0m best_strategies \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbuy_and_hold\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[43mbuyandhold_best_strategies\u001b[49m,\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgmadl_model\u001b[39m\u001b[38;5;124m'\u001b[39m: gmadl_model_best_strategies\n\u001b[1;32m 5\u001b[0m }\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcache/1min-best-strategies-v1.pkl\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m outp:\n\u001b[1;32m 8\u001b[0m pickle\u001b[38;5;241m.\u001b[39mdump(best_strategies, outp, pickle\u001b[38;5;241m.\u001b[39mHIGHEST_PROTOCOL)\n", "\u001b[0;31mNameError\u001b[0m: name 'buyandhold_best_strategies' is not defined" ] } ], "source": [ "# Persist best strategies, so that they don't have to be recomputed every time\n", "best_strategies = {\n", " 'buy_and_hold': buyandhold_best_strategies,\n", " 'gmadl_model': gmadl_model_best_strategies\n", "}\n", "\n", "with open('cache/1min-best-strategies-v1.pkl', 'wb') as outp:\n", " pickle.dump(best_strategies, outp, pickle.HIGHEST_PROTOCOL)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "with open('cache/1min-best-strategies-v1.pkl', 'rb') as inpt:\n", " best_strategies = pickle.load(inpt)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# plot_sweep_results(pd.DataFrame([result for result, _ in gmadl_model_sweep_results[0]]), parameters=['enter_long', 'exit_long', 'enter_short', 'exit_short'], round=5, objective='mod_ir')" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def results_plot(idx, result_buyandhold, result_gmadl_model, width=850, height=500, notitle=False, v_lines=None):\n", "\n", " fig = go.Figure([\n", " go.Scatter(y=result_buyandhold['portfolio_value'], x=result_buyandhold['time'], name=\"Buy and Hold\"),\n", " go.Scatter(y=result_gmadl_model['portfolio_value'], x=result_gmadl_model['time'], name='GMADL Informer Strategy')\n", " ])\n", " \n", " if v_lines:\n", " for v_line in v_lines:\n", " fig.add_shape(\n", " go.layout.Shape(type=\"line\",\n", " yref=\"paper\",\n", " xref=\"x\",\n", " x0=v_line,\n", " x1=v_line,\n", " y0=0,\n", " y1=1,\n", " line=dict(dash='dash', color='rgb(140,140,140)')))\n", " fig.update_layout(\n", " title={\n", " 'text': f\"W{idx}-{INTERVAL}\",\n", " 'y':0.97,\n", " 'x':0.5,\n", " 'xanchor': 'center',\n", " 'yanchor': 'top'} if not notitle else None,\n", " yaxis_title=\"Portfolio Value\",\n", " xaxis_title=\"Date\",\n", " font=dict(\n", " # family=\"Courier New, monospace\",\n", " size=14,\n", " ),\n", " autosize=False,\n", " width=width,\n", " height=height,\n", " margin=dict(l=20, r=20, t=20 if notitle else 110, b=20),\n", " plot_bgcolor='white',\n", " legend=dict(\n", " orientation=\"h\",\n", " yanchor=\"bottom\",\n", " y=1.02,\n", " xanchor=\"left\",\n", " x=0.02\n", " )\n", " )\n", " fig.update_xaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " fig.update_yaxes(\n", " mirror=True,\n", " ticks='outside',\n", " showline=True,\n", " linecolor='black',\n", " gridcolor='lightgrey'\n", " )\n", " fig.write_image(f\"images/eval-w{idx}-{INTERVAL}.png\")\n", " fig.show()\n", " \n", "def results_table(result_buyandhold, result_gmadl_model):\n", " table_eval_windows = Texttable()\n", " table_eval_windows.set_deco(Texttable.HEADER)\n", " table_eval_windows.set_cols_align([\"l\", \"c\",\"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\", \"c\"])\n", " table_eval_windows.set_precision(3)\n", "\n", " table_eval_windows.header([\n", " \"\\\\textbf{Strategy}\",\n", " \"\\\\textbf{VAL}\",\n", " \"\\\\textbf{ARC}\",\n", " \"\\\\textbf{ASD}\",\n", " \"\\\\textbf{IR*}\",\n", " \"\\\\textbf{MD}\",\n", " \"\\\\textbf{IR**}\",\n", " \"\\\\textbf{N}\",\n", " \"\\\\textbf{LONG}\",\n", " \"\\\\textbf{SHORT}\",\n", " ])\n", "\n", " strategy_name_result = [\n", " ('Buy and Hold', result_buyandhold),\n", " ('GMADL Informer', result_gmadl_model)\n", " ]\n", " for strategy_name, result in strategy_name_result:\n", " table_eval_windows.add_row([\n", " strategy_name,\n", " result['value'],\n", " f\"{result['arc']*100:.2f}\\%\",\n", " f\"{result['asd']*100:.2f}\\%\",\n", " result['ir'],\n", " f\"{result['md']*100:.2f}\\%\",\n", " result['mod_ir'],\n", " result['n_trades'],\n", " f\"{result['long_pos']*100:.2f}\\%\",\n", " f\"{result['short_pos']*100:.2f}\\%\",\n", " ])\n", " print(latextable.draw_latex(table_eval_windows))\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 0.929 & -13.87\\% & 69.66\\% & -0.199 & 52.09\\% & -0.053 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 1.306 & 71.83\\% & 69.69\\% & 1.031 & 41.57\\% & 1.781 & 50 & 7.29\\% & 92.71\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n", "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 0.549 & -70.35\\% & 73.36\\% & -0.959 & 63.40\\% & -1.064 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 1.837 & 243.15\\% & 73.38\\% & 3.314 & 25.16\\% & 32.024 & 186 & 18.19\\% & 81.81\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n", "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 1.016 & 3.33\\% & 52.45\\% & 0.064 & 38.42\\% & 0.006 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 0.739 & -45.82\\% & 52.21\\% & -0.878 & 42.46\\% & -0.947 & 35 & 4.70\\% & 93.05\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n", "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 1.230 & 52.29\\% & 44.30\\% & 1.180 & 22.35\\% & 2.761 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 1.086 & 18.12\\% & 40.58\\% & 0.446 & 26.30\\% & 0.308 & 11 & 60.03\\% & 23.82\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n", "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 1.439 & 109.31\\% & 43.75\\% & 2.498 & 21.12\\% & 12.930 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 1.010 & 1.98\\% & 43.47\\% & 0.046 & 31.96\\% & 0.003 & 67 & 80.24\\% & 15.37\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n", "\\begin{table}\n", "\t\\begin{center}\n", "\t\t\\begin{tabular}{lccccccccc}\n", "\t\t\t\\textbf{Strategy} & \\textbf{VAL} & \\textbf{ARC} & \\textbf{ASD} & \\textbf{IR*} & \\textbf{MD} & \\textbf{IR**} & \\textbf{N} & \\textbf{LONG} & \\textbf{SHORT} \\\\\n", "\t\t\t\\hline\n", "\t\t\tBuy and Hold & 1.561 & 146.58\\% & 53.72\\% & 2.729 & 27.11\\% & 14.756 & 2 & 100.00\\% & 0.00\\% \\\\\n", "\t\t\tGMADL Informer & 1.178 & 39.32\\% & 43.06\\% & 0.913 & 18.63\\% & 1.927 & 92 & 54.86\\% & 0.00\\% \\\\\n", "\t\t\\end{tabular}\n", "\t\\end{center}\n", "\\end{table}\n" ] } ], "source": [ "for i, (in_sample, out_of_sample) in enumerate(data_windows):\n", " padded_window = pd.concat([in_sample.iloc[-PADDING:], out_of_sample])\n", " result_buyandhold = evaluate_strategy(padded_window, best_strategies['buy_and_hold'][i], padding=PADDING, interval=INTERVAL)\n", " result_gmadl_model = evaluate_strategy(padded_window, [s[0] for s in best_strategies['gmadl_model']][i], padding=PADDING, interval=INTERVAL)\n", "\n", " results_table(result_buyandhold, result_gmadl_model)\n", " # results_plot(i+1, result_buyandhold, result_gmadl_model)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# test_data = pd.concat([data_windows[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows])\n", "# buy_and_hold_concat = evaluate_strategy(test_data, BuyAndHoldStrategy(), padding=PADDING, interval=INTERVAL)\n", "# gmadl_model_concat = evaluate_strategy(test_data, ConcatenatedStrategies(len(data_windows[0][1]), [s[0] for s in best_strategies['gmadl_model']], padding=PADDING), padding=PADDING, interval=INTERVAL)\n", "\n", "# v_lines=[data_window[1]['close_time'].iloc[-1] for data_window in data_windows][:-1]\n", "# results_table(buy_and_hold_concat, gmadl_model_concat)\n", "# results_plot(0, buy_and_hold_concat, gmadl_model_concat, width=1300, height=500, notitle=True)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import plotly.figure_factory as ff\n", "\n", "def results_for_strats(\n", " data_windows, \n", " best_strategies,\n", " top_n=10):\n", " test_data = pd.concat([data_windows[0][0][-PADDING:]] + [data_window[1] for data_window in data_windows])\n", "\n", " buy_and_hold_concat = evaluate_strategy(test_data, BuyAndHoldStrategy(), padding=PADDING, interval='min')\n", " gmadl_1min_model_concat = [evaluate_strategy(test_data, ConcatenatedStrategies(len(data_windows[0][1]), [s[x] for s in best_strategies['gmadl_model']], padding=PADDING), padding=PADDING, interval='min') for x in range(top_n)]\n", "\n", " z = list(reversed([\n", " list(reversed([round(buy_and_hold_concat['mod_ir'], 3)]*top_n)),\n", " list(reversed([round(x['mod_ir'], 3) for x in gmadl_1min_model_concat])),\n", " ]))\n", " x = list(reversed(range(1, top_n+1)))\n", " y = list(reversed([\n", " \"Buy and Hold\",\n", " \"Gmadl Informer (1 min)\"\n", " ]))\n", " # 'Portland'\n", " fig = ff.create_annotated_heatmap(z, x=x, y=y, colorscale='thermal', zmid=buy_and_hold_concat['mod_ir'])\n", " fig.update_layout(\n", " margin=dict(l=20, r=20, b=20, t=20),\n", " width=1100,\n", " height=650,\n", " font=dict(\n", " # family=\"Courier New, monospace\",\n", " size=16, # Set the font size here\n", " # color=\"RebeccaPurple\"\n", " )\n", " )\n", " fig.show()\n", "\n", "# results_for_strats(data_windows, best_strategies, top_n=10) " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "wnemsc", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.19" } }, "nbformat": 4, "nbformat_minor": 2 }