{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1023f2c1-e45f-4e1c-9a1b-66f59f128196",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Panda Version: 2.2.2\n",
      "Today date is:  2024-06-06\n"
     ]
    }
   ],
   "source": [
    "import sqlite3\n",
    "\n",
    "import numpy as np\n",
    "# It is apparently officially accepted to explicitly\n",
    "# list all the functions you need from numpy:\n",
    "from numpy import array, zeros, exp, random, dot\n",
    "from numpy import shape, reshape, meshgrid, linspace\n",
    "from numpy import hstack, vstack\n",
    "\n",
    "import pandas as pd\n",
    "print('Panda Version:', pd.__version__)\n",
    "\n",
    "# Set ipython's max row display\n",
    "pd.set_option('display.max_row', 100, 'display.max_columns', 25)\n",
    "\n",
    "import matplotlib.pyplot as plt # for plotting\n",
    "import matplotlib\n",
    "matplotlib.rcParams['figure.dpi'] = 100 # highres display\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import Variable\n",
    "\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.layers import TimeDistributed, RepeatVector\n",
    "\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "from keras.models import load_model\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "from keras.layers import LSTM, Dense, Concatenate\n",
    "\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "import collections\n",
    "from collections import Counter\n",
    "\n",
    "# Import date class from datetime module\n",
    "import time\n",
    "# import datetime as dt\n",
    "from datetime import date, datetime\n",
    "print(\"Today date is: \", date.today())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c09a37a6-f0d9-48e3-a1d1-65ddaf2c489c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/leo\n",
      "total 6528\n",
      "drwxrwxr-x 1 oleg oleg     250 Jun  6 10:00 .\n",
      "drwxrwxr-x 1 oleg oleg      18 Jun  3 23:40 ..\n",
      "-rw------- 1 oleg oleg 1449984 Jun  4 00:49 20240601.mktdata.ohlcv.db\n",
      "-rw------- 1 oleg oleg 1445888 Jun  3 23:44 20240602.mktdata.ohlcv.db\n",
      "-rw------- 1 oleg oleg 1437696 Jun  4 16:45 20240603.mktdata.ohlcv.db\n",
      "-rw------- 1 oleg oleg 1269760 Jun  5 10:00 20240604.mktdata.ohlcv.db\n",
      "-rw------- 1 oleg oleg 1081344 Jun  6 10:00 20240605.mktdata.ohlcv.db\n"
     ]
    }
   ],
   "source": [
    "!pwd\n",
    "\n",
    "!ls -la /workspace/data/crypto_md/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d72d849-e8df-4564-a006-03ab646b9330",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "58d263ae-d028-4de0-9f10-0f14c4de28f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "mktdata_db_file = \"/workspace/data/crypto_md/20240601.mktdata.ohlcv.db\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70033f48-f283-43f3-8e86-3bd533a10fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "db_conn = sqlite3.connect(mktdata_db_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2cb82c91-a487-46b1-bcd0-012f7675010f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    type                 name             tbl_name  rootpage  \\\n",
      "0  table    bnbfut_ohlcv_1min    bnbfut_ohlcv_1min         2   \n",
      "1  table   bnbspot_ohlcv_1min   bnbspot_ohlcv_1min        66   \n",
      "2  table  coinbase_ohlcv_1min  coinbase_ohlcv_1min       224   \n",
      "\n",
      "                                                 sql  \n",
      "0  CREATE TABLE bnbfut_ohlcv_1min (tstamp INTEGER...  \n",
      "1  CREATE TABLE bnbspot_ohlcv_1min (tstamp INTEGE...  \n",
      "2  CREATE TABLE coinbase_ohlcv_1min (tstamp INTEG...  \n"
     ]
    }
   ],
   "source": [
    "tables_df = pd.read_sql_query(\"select * from sqlite_master where type = 'table'\", db_conn)\n",
    "print (tables_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cf159bde-1ccc-40e0-9eb9-a010c1372e07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tstamp</th>\n",
       "      <th>exchange_id</th>\n",
       "      <th>instrument_id</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "      <th>vwap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5754</th>\n",
       "      <td>1717286100000000000</td>\n",
       "      <td>COINBASE</td>\n",
       "      <td>PAIR-XRP-USD</td>\n",
       "      <td>0.5183</td>\n",
       "      <td>0.5183</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>7401.243996</td>\n",
       "      <td>0.518096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5755</th>\n",
       "      <td>1717286160000000000</td>\n",
       "      <td>COINBASE</td>\n",
       "      <td>PAIR-XRP-USD</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>0.5182</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>3491.632415</td>\n",
       "      <td>0.518020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5756</th>\n",
       "      <td>1717286220000000000</td>\n",
       "      <td>COINBASE</td>\n",
       "      <td>PAIR-XRP-USD</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>0.5180</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>6963.191620</td>\n",
       "      <td>0.518095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5757</th>\n",
       "      <td>1717286280000000000</td>\n",
       "      <td>COINBASE</td>\n",
       "      <td>PAIR-XRP-USD</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>0.5183</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>0.5182</td>\n",
       "      <td>35743.462893</td>\n",
       "      <td>0.518140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5758</th>\n",
       "      <td>1717286340000000000</td>\n",
       "      <td>COINBASE</td>\n",
       "      <td>PAIR-XRP-USD</td>\n",
       "      <td>0.5182</td>\n",
       "      <td>0.5184</td>\n",
       "      <td>0.5181</td>\n",
       "      <td>0.5184</td>\n",
       "      <td>45521.897366</td>\n",
       "      <td>0.518245</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   tstamp exchange_id instrument_id    open    high     low  \\\n",
       "5754  1717286100000000000    COINBASE  PAIR-XRP-USD  0.5183  0.5183  0.5180   \n",
       "5755  1717286160000000000    COINBASE  PAIR-XRP-USD  0.5180  0.5182  0.5180   \n",
       "5756  1717286220000000000    COINBASE  PAIR-XRP-USD  0.5181  0.5181  0.5180   \n",
       "5757  1717286280000000000    COINBASE  PAIR-XRP-USD  0.5181  0.5183  0.5181   \n",
       "5758  1717286340000000000    COINBASE  PAIR-XRP-USD  0.5182  0.5184  0.5181   \n",
       "\n",
       "       close        volume      vwap  \n",
       "5754  0.5180   7401.243996  0.518096  \n",
       "5755  0.5180   3491.632415  0.518020  \n",
       "5756  0.5181   6963.191620  0.518095  \n",
       "5757  0.5182  35743.462893  0.518140  \n",
       "5758  0.5184  45521.897366  0.518245  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11d73b0a-fe9b-4d73-87c3-4250ef98b16a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0601 - 0605"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd83027c-eaa1-44d9-98ad-8506099c323d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ccc60bc-6b74-408c-946c-bdbfee0cffb5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f358ee5-9ec0-4582-be09-4e4ad84faca7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fab70130-9661-4efd-ad45-7647d81cc6ce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "77504183-34d8-42c6-93a3-d86a34388f9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                  tstamp exchange_id instrument_id      open  \\\n",
      "timestamp                                                                      \n",
      "2024-06-01 00:00:00  1717200000000000000    COINBASE  PAIR-BTC-USD  67473.07   \n",
      "2024-06-01 00:01:00  1717200060000000000    COINBASE  PAIR-BTC-USD  67506.57   \n",
      "2024-06-01 00:02:00  1717200120000000000    COINBASE  PAIR-BTC-USD  67494.73   \n",
      "2024-06-01 00:03:00  1717200180000000000    COINBASE  PAIR-BTC-USD  67512.78   \n",
      "2024-06-01 00:04:00  1717200240000000000    COINBASE  PAIR-BTC-USD  67538.81   \n",
      "\n",
      "                         high       low     close    volume          vwap  \n",
      "timestamp                                                                  \n",
      "2024-06-01 00:00:00  67514.99  67468.13  67506.03  2.045049  67496.869352  \n",
      "2024-06-01 00:01:00  67515.00  67480.66  67494.74  0.646759  67500.893305  \n",
      "2024-06-01 00:02:00  67515.00  67455.91  67514.99  9.732906  67477.840770  \n",
      "2024-06-01 00:03:00  67553.85  67496.58  67539.76  4.374730  67522.912057  \n",
      "2024-06-01 00:04:00  67579.13  67530.05  67579.13  2.215383  67557.086208  \n"
     ]
    }
   ],
   "source": [
    "df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n",
    "df.set_index(\"timestamp\", inplace=True)\n",
    "print (df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "316c0cfb-c73d-4dad-9d49-eb80daa229ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92700c76-8eac-4ebb-86d3-27066486c437",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3604f3f2-3e56-477d-a7ac-d6490839642d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_union = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7c8b332-cd4a-455f-b7cf-381aec15c456",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a27972a-f457-4ca5-8530-d6c87c7d9d91",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5356525-e614-4858-af4e-648c03b7d21e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "695de24a-41f4-4c82-a0dd-dbbd7bfba2bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c6d80b3-4a46-432b-9b88-6a264f9b7a7e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
