{ "cells": [ { "cell_type": "code", "execution_count": 4, "id": "1023f2c1-e45f-4e1c-9a1b-66f59f128196", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Panda Version: 2.2.2\n", "Today date is: 2024-06-06\n" ] } ], "source": [ "import sqlite3\n", "\n", "import numpy as np\n", "# It is apparently officially accepted to explicitly\n", "# list all the functions you need from numpy:\n", "from numpy import array, zeros, exp, random, dot\n", "from numpy import shape, reshape, meshgrid, linspace\n", "from numpy import hstack, vstack\n", "\n", "import pandas as pd\n", "print('Panda Version:', pd.__version__)\n", "\n", "# Set ipython's max row display\n", "pd.set_option('display.max_row', 100, 'display.max_columns', 25)\n", "\n", "import matplotlib.pyplot as plt # for plotting\n", "import matplotlib\n", "matplotlib.rcParams['figure.dpi'] = 100 # highres display\n", "\n", "import tensorflow as tf\n", "from tensorflow import Variable\n", "\n", "from tensorflow.keras import Sequential\n", "from tensorflow.keras import Model\n", "from tensorflow.keras.layers import Dense\n", "from tensorflow.keras.layers import Dropout\n", "from tensorflow.keras.layers import TimeDistributed, RepeatVector\n", "\n", "from keras.optimizers import SGD\n", "\n", "from keras.models import load_model\n", "from keras.callbacks import EarlyStopping\n", "from keras.callbacks import ModelCheckpoint\n", "\n", "from keras.layers import LSTM, Dense, Concatenate\n", "\n", "from keras.optimizers import SGD\n", "\n", "import collections\n", "from collections import Counter\n", "\n", "# Import date class from datetime module\n", "import time\n", "# import datetime as dt\n", "from datetime import date, datetime\n", "print(\"Today date is: \", date.today())" ] }, { "cell_type": "code", "execution_count": 28, "id": "c09a37a6-f0d9-48e3-a1d1-65ddaf2c489c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/workspace/leo\n", "total 6528\n", "drwxrwxr-x 1 oleg oleg 250 Jun 6 10:00 .\n", "drwxrwxr-x 1 oleg oleg 18 Jun 3 23:40 ..\n", "-rw------- 1 oleg oleg 1449984 Jun 4 00:49 20240601.mktdata.ohlcv.db\n", "-rw------- 1 oleg oleg 1445888 Jun 3 23:44 20240602.mktdata.ohlcv.db\n", "-rw------- 1 oleg oleg 1437696 Jun 4 16:45 20240603.mktdata.ohlcv.db\n", "-rw------- 1 oleg oleg 1269760 Jun 5 10:00 20240604.mktdata.ohlcv.db\n", "-rw------- 1 oleg oleg 1081344 Jun 6 10:00 20240605.mktdata.ohlcv.db\n" ] } ], "source": [ "!pwd\n", "\n", "!ls -la /workspace/data/crypto_md/" ] }, { "cell_type": "code", "execution_count": null, "id": "8d72d849-e8df-4564-a006-03ab646b9330", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 24, "id": "58d263ae-d028-4de0-9f10-0f14c4de28f7", "metadata": {}, "outputs": [], "source": [ "mktdata_db_file = \"/workspace/data/crypto_md/20240601.mktdata.ohlcv.db\"" ] }, { "cell_type": "code", "execution_count": null, "id": "70033f48-f283-43f3-8e86-3bd533a10fed", "metadata": {}, "outputs": [], "source": [ "db_conn = sqlite3.connect(mktdata_db_file)" ] }, { "cell_type": "code", "execution_count": 31, "id": "2cb82c91-a487-46b1-bcd0-012f7675010f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " type name tbl_name rootpage \\\n", "0 table bnbfut_ohlcv_1min bnbfut_ohlcv_1min 2 \n", "1 table bnbspot_ohlcv_1min bnbspot_ohlcv_1min 66 \n", "2 table coinbase_ohlcv_1min coinbase_ohlcv_1min 224 \n", "\n", " sql \n", "0 CREATE TABLE bnbfut_ohlcv_1min (tstamp INTEGER... \n", "1 CREATE TABLE bnbspot_ohlcv_1min (tstamp INTEGE... \n", "2 CREATE TABLE coinbase_ohlcv_1min (tstamp INTEG... \n" ] } ], "source": [ "tables_df = pd.read_sql_query(\"select * from sqlite_master where type = 'table'\", db_conn)\n", "print (tables_df.head())" ] }, { "cell_type": "code", "execution_count": 34, "id": "cf159bde-1ccc-40e0-9eb9-a010c1372e07", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
57541717286100000000000COINBASEPAIR-XRP-USD0.51830.51830.51800.51807401.2439960.518096
57551717286160000000000COINBASEPAIR-XRP-USD0.51800.51820.51800.51803491.6324150.518020
57561717286220000000000COINBASEPAIR-XRP-USD0.51810.51810.51800.51816963.1916200.518095
57571717286280000000000COINBASEPAIR-XRP-USD0.51810.51830.51810.518235743.4628930.518140
57581717286340000000000COINBASEPAIR-XRP-USD0.51820.51840.51810.518445521.8973660.518245
\n", "
" ], "text/plain": [ " tstamp exchange_id instrument_id open high low \\\n", "5754 1717286100000000000 COINBASE PAIR-XRP-USD 0.5183 0.5183 0.5180 \n", "5755 1717286160000000000 COINBASE PAIR-XRP-USD 0.5180 0.5182 0.5180 \n", "5756 1717286220000000000 COINBASE PAIR-XRP-USD 0.5181 0.5181 0.5180 \n", "5757 1717286280000000000 COINBASE PAIR-XRP-USD 0.5181 0.5183 0.5181 \n", "5758 1717286340000000000 COINBASE PAIR-XRP-USD 0.5182 0.5184 0.5181 \n", "\n", " close volume vwap \n", "5754 0.5180 7401.243996 0.518096 \n", "5755 0.5180 3491.632415 0.518020 \n", "5756 0.5181 6963.191620 0.518095 \n", "5757 0.5182 35743.462893 0.518140 \n", "5758 0.5184 45521.897366 0.518245 " ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)\n", "df.tail()" ] }, { "cell_type": "code", "execution_count": null, "id": "11d73b0a-fe9b-4d73-87c3-4250ef98b16a", "metadata": {}, "outputs": [], "source": [ "# 0601 - 0605" ] }, { "cell_type": "code", "execution_count": null, "id": "cd83027c-eaa1-44d9-98ad-8506099c323d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6ccc60bc-6b74-408c-946c-bdbfee0cffb5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0f358ee5-9ec0-4582-be09-4e4ad84faca7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fab70130-9661-4efd-ad45-7647d81cc6ce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 33, "id": "77504183-34d8-42c6-93a3-d86a34388f9e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " tstamp exchange_id instrument_id open \\\n", "timestamp \n", "2024-06-01 00:00:00 1717200000000000000 COINBASE PAIR-BTC-USD 67473.07 \n", "2024-06-01 00:01:00 1717200060000000000 COINBASE PAIR-BTC-USD 67506.57 \n", "2024-06-01 00:02:00 1717200120000000000 COINBASE PAIR-BTC-USD 67494.73 \n", "2024-06-01 00:03:00 1717200180000000000 COINBASE PAIR-BTC-USD 67512.78 \n", "2024-06-01 00:04:00 1717200240000000000 COINBASE PAIR-BTC-USD 67538.81 \n", "\n", " high low close volume vwap \n", "timestamp \n", "2024-06-01 00:00:00 67514.99 67468.13 67506.03 2.045049 67496.869352 \n", "2024-06-01 00:01:00 67515.00 67480.66 67494.74 0.646759 67500.893305 \n", "2024-06-01 00:02:00 67515.00 67455.91 67514.99 9.732906 67477.840770 \n", "2024-06-01 00:03:00 67553.85 67496.58 67539.76 4.374730 67522.912057 \n", "2024-06-01 00:04:00 67579.13 67530.05 67579.13 2.215383 67557.086208 \n" ] } ], "source": [ "df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n", "df.set_index(\"timestamp\", inplace=True)\n", "print (df.head())" ] }, { "cell_type": "code", "execution_count": null, "id": "316c0cfb-c73d-4dad-9d49-eb80daa229ec", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "92700c76-8eac-4ebb-86d3-27066486c437", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3604f3f2-3e56-477d-a7ac-d6490839642d", "metadata": {}, "outputs": [], "source": [ "df_union = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)" ] }, { "cell_type": "code", "execution_count": null, "id": "a7c8b332-cd4a-455f-b7cf-381aec15c456", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0a27972a-f457-4ca5-8530-d6c87c7d9d91", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b5356525-e614-4858-af4e-648c03b7d21e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "695de24a-41f4-4c82-a0dd-dbbd7bfba2bd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3c6d80b3-4a46-432b-9b88-6a264f9b7a7e", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0rc1" } }, "nbformat": 4, "nbformat_minor": 5 }