444 lines
13 KiB
Plaintext
444 lines
13 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "1023f2c1-e45f-4e1c-9a1b-66f59f128196",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Panda Version: 2.2.2\n",
|
|
"Today date is: 2024-06-06\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import sqlite3\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"# It is apparently officially accepted to explicitly\n",
|
|
"# list all the functions you need from numpy:\n",
|
|
"from numpy import array, zeros, exp, random, dot\n",
|
|
"from numpy import shape, reshape, meshgrid, linspace\n",
|
|
"from numpy import hstack, vstack\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"print('Panda Version:', pd.__version__)\n",
|
|
"\n",
|
|
"# Set ipython's max row display\n",
|
|
"pd.set_option('display.max_row', 100, 'display.max_columns', 25)\n",
|
|
"\n",
|
|
"import matplotlib.pyplot as plt # for plotting\n",
|
|
"import matplotlib\n",
|
|
"matplotlib.rcParams['figure.dpi'] = 100 # highres display\n",
|
|
"\n",
|
|
"import tensorflow as tf\n",
|
|
"from tensorflow import Variable\n",
|
|
"\n",
|
|
"from tensorflow.keras import Sequential\n",
|
|
"from tensorflow.keras import Model\n",
|
|
"from tensorflow.keras.layers import Dense\n",
|
|
"from tensorflow.keras.layers import Dropout\n",
|
|
"from tensorflow.keras.layers import TimeDistributed, RepeatVector\n",
|
|
"\n",
|
|
"from keras.optimizers import SGD\n",
|
|
"\n",
|
|
"from keras.models import load_model\n",
|
|
"from keras.callbacks import EarlyStopping\n",
|
|
"from keras.callbacks import ModelCheckpoint\n",
|
|
"\n",
|
|
"from keras.layers import LSTM, Dense, Concatenate\n",
|
|
"\n",
|
|
"from keras.optimizers import SGD\n",
|
|
"\n",
|
|
"import collections\n",
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"# Import date class from datetime module\n",
|
|
"import time\n",
|
|
"# import datetime as dt\n",
|
|
"from datetime import date, datetime\n",
|
|
"print(\"Today date is: \", date.today())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"id": "c09a37a6-f0d9-48e3-a1d1-65ddaf2c489c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/workspace/leo\n",
|
|
"total 6528\n",
|
|
"drwxrwxr-x 1 oleg oleg 250 Jun 6 10:00 .\n",
|
|
"drwxrwxr-x 1 oleg oleg 18 Jun 3 23:40 ..\n",
|
|
"-rw------- 1 oleg oleg 1449984 Jun 4 00:49 20240601.mktdata.ohlcv.db\n",
|
|
"-rw------- 1 oleg oleg 1445888 Jun 3 23:44 20240602.mktdata.ohlcv.db\n",
|
|
"-rw------- 1 oleg oleg 1437696 Jun 4 16:45 20240603.mktdata.ohlcv.db\n",
|
|
"-rw------- 1 oleg oleg 1269760 Jun 5 10:00 20240604.mktdata.ohlcv.db\n",
|
|
"-rw------- 1 oleg oleg 1081344 Jun 6 10:00 20240605.mktdata.ohlcv.db\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"!pwd\n",
|
|
"\n",
|
|
"!ls -la /workspace/data/crypto_md/"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8d72d849-e8df-4564-a006-03ab646b9330",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 24,
|
|
"id": "58d263ae-d028-4de0-9f10-0f14c4de28f7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"mktdata_db_file = \"/workspace/data/crypto_md/20240601.mktdata.ohlcv.db\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "70033f48-f283-43f3-8e86-3bd533a10fed",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"db_conn = sqlite3.connect(mktdata_db_file)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 31,
|
|
"id": "2cb82c91-a487-46b1-bcd0-012f7675010f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" type name tbl_name rootpage \\\n",
|
|
"0 table bnbfut_ohlcv_1min bnbfut_ohlcv_1min 2 \n",
|
|
"1 table bnbspot_ohlcv_1min bnbspot_ohlcv_1min 66 \n",
|
|
"2 table coinbase_ohlcv_1min coinbase_ohlcv_1min 224 \n",
|
|
"\n",
|
|
" sql \n",
|
|
"0 CREATE TABLE bnbfut_ohlcv_1min (tstamp INTEGER... \n",
|
|
"1 CREATE TABLE bnbspot_ohlcv_1min (tstamp INTEGE... \n",
|
|
"2 CREATE TABLE coinbase_ohlcv_1min (tstamp INTEG... \n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"tables_df = pd.read_sql_query(\"select * from sqlite_master where type = 'table'\", db_conn)\n",
|
|
"print (tables_df.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 34,
|
|
"id": "cf159bde-1ccc-40e0-9eb9-a010c1372e07",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>tstamp</th>\n",
|
|
" <th>exchange_id</th>\n",
|
|
" <th>instrument_id</th>\n",
|
|
" <th>open</th>\n",
|
|
" <th>high</th>\n",
|
|
" <th>low</th>\n",
|
|
" <th>close</th>\n",
|
|
" <th>volume</th>\n",
|
|
" <th>vwap</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>5754</th>\n",
|
|
" <td>1717286100000000000</td>\n",
|
|
" <td>COINBASE</td>\n",
|
|
" <td>PAIR-XRP-USD</td>\n",
|
|
" <td>0.5183</td>\n",
|
|
" <td>0.5183</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>7401.243996</td>\n",
|
|
" <td>0.518096</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5755</th>\n",
|
|
" <td>1717286160000000000</td>\n",
|
|
" <td>COINBASE</td>\n",
|
|
" <td>PAIR-XRP-USD</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>0.5182</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>3491.632415</td>\n",
|
|
" <td>0.518020</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5756</th>\n",
|
|
" <td>1717286220000000000</td>\n",
|
|
" <td>COINBASE</td>\n",
|
|
" <td>PAIR-XRP-USD</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>0.5180</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>6963.191620</td>\n",
|
|
" <td>0.518095</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5757</th>\n",
|
|
" <td>1717286280000000000</td>\n",
|
|
" <td>COINBASE</td>\n",
|
|
" <td>PAIR-XRP-USD</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>0.5183</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>0.5182</td>\n",
|
|
" <td>35743.462893</td>\n",
|
|
" <td>0.518140</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5758</th>\n",
|
|
" <td>1717286340000000000</td>\n",
|
|
" <td>COINBASE</td>\n",
|
|
" <td>PAIR-XRP-USD</td>\n",
|
|
" <td>0.5182</td>\n",
|
|
" <td>0.5184</td>\n",
|
|
" <td>0.5181</td>\n",
|
|
" <td>0.5184</td>\n",
|
|
" <td>45521.897366</td>\n",
|
|
" <td>0.518245</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" tstamp exchange_id instrument_id open high low \\\n",
|
|
"5754 1717286100000000000 COINBASE PAIR-XRP-USD 0.5183 0.5183 0.5180 \n",
|
|
"5755 1717286160000000000 COINBASE PAIR-XRP-USD 0.5180 0.5182 0.5180 \n",
|
|
"5756 1717286220000000000 COINBASE PAIR-XRP-USD 0.5181 0.5181 0.5180 \n",
|
|
"5757 1717286280000000000 COINBASE PAIR-XRP-USD 0.5181 0.5183 0.5181 \n",
|
|
"5758 1717286340000000000 COINBASE PAIR-XRP-USD 0.5182 0.5184 0.5181 \n",
|
|
"\n",
|
|
" close volume vwap \n",
|
|
"5754 0.5180 7401.243996 0.518096 \n",
|
|
"5755 0.5180 3491.632415 0.518020 \n",
|
|
"5756 0.5181 6963.191620 0.518095 \n",
|
|
"5757 0.5182 35743.462893 0.518140 \n",
|
|
"5758 0.5184 45521.897366 0.518245 "
|
|
]
|
|
},
|
|
"execution_count": 34,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)\n",
|
|
"df.tail()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "11d73b0a-fe9b-4d73-87c3-4250ef98b16a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# 0601 - 0605"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cd83027c-eaa1-44d9-98ad-8506099c323d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6ccc60bc-6b74-408c-946c-bdbfee0cffb5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0f358ee5-9ec0-4582-be09-4e4ad84faca7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fab70130-9661-4efd-ad45-7647d81cc6ce",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 33,
|
|
"id": "77504183-34d8-42c6-93a3-d86a34388f9e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" tstamp exchange_id instrument_id open \\\n",
|
|
"timestamp \n",
|
|
"2024-06-01 00:00:00 1717200000000000000 COINBASE PAIR-BTC-USD 67473.07 \n",
|
|
"2024-06-01 00:01:00 1717200060000000000 COINBASE PAIR-BTC-USD 67506.57 \n",
|
|
"2024-06-01 00:02:00 1717200120000000000 COINBASE PAIR-BTC-USD 67494.73 \n",
|
|
"2024-06-01 00:03:00 1717200180000000000 COINBASE PAIR-BTC-USD 67512.78 \n",
|
|
"2024-06-01 00:04:00 1717200240000000000 COINBASE PAIR-BTC-USD 67538.81 \n",
|
|
"\n",
|
|
" high low close volume vwap \n",
|
|
"timestamp \n",
|
|
"2024-06-01 00:00:00 67514.99 67468.13 67506.03 2.045049 67496.869352 \n",
|
|
"2024-06-01 00:01:00 67515.00 67480.66 67494.74 0.646759 67500.893305 \n",
|
|
"2024-06-01 00:02:00 67515.00 67455.91 67514.99 9.732906 67477.840770 \n",
|
|
"2024-06-01 00:03:00 67553.85 67496.58 67539.76 4.374730 67522.912057 \n",
|
|
"2024-06-01 00:04:00 67579.13 67530.05 67579.13 2.215383 67557.086208 \n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n",
|
|
"df.set_index(\"timestamp\", inplace=True)\n",
|
|
"print (df.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "316c0cfb-c73d-4dad-9d49-eb80daa229ec",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "92700c76-8eac-4ebb-86d3-27066486c437",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3604f3f2-3e56-477d-a7ac-d6490839642d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_union = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a7c8b332-cd4a-455f-b7cf-381aec15c456",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0a27972a-f457-4ca5-8530-d6c87c7d9d91",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b5356525-e614-4858-af4e-648c03b7d21e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "695de24a-41f4-4c82-a0dd-dbbd7bfba2bd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3c6d80b3-4a46-432b-9b88-6a264f9b7a7e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.0rc1"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|