progress

.gitignore

@@ -7,3 +7,4 @@
 data
 *webp
 jupyter_gpu.code-workspace
+cvtt

pytorch/.gitignore

@@ -0,0 +1 @@
+**/AutogluonModels/

pytorch/docker-compose.yml

@@ -8,17 +8,19 @@ version: '3.8'
 services:
   jupyter:
     build: ${ROOT_DIR}
-    image: cvtt/jupyter_gpu_pt:v1.1.2
+    image: cvtt/jupyter_gpu_pt:v1.1.3
     container_name: jupyter_gpu_pt
     runtime: nvidia
     environment:
       - JUPYTER_ENABLE_LAB=yes
+      - PYTHONPATH=/cvtt/prod
     volumes:
       - ${ROOT_DIR}/notebooks:/workspace
       - ${ROOT_DIR}/jupyter_settings/user-settings:/root/.jupyter/lab/user-settings
       - ${ROOT_DIR}/jupyter_settings/workspaces:/root/.jupyter/lab/workspaces
       - ${ROOT_DIR}/.cache/torch:/root/.cache/torch
       - /opt/jupyter_gpu/data:/workspace/data
+      - ${ROOT_DIR}/cvtt:/cvtt
     ports:
       - "${JUPYTER_PORT}:${JUPYTER_PORT}"
     shm_size: "8gb"

pytorch/get_version.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+function usage() {
+    echo "Usage: ${0} <project> <version>"
+    exit 1
+
+}
+
+# ----- Settings
+Root=/opt/jupyter_gpu/pytorch/cvtt
+
+LocalSoftwareDir=${Root}/software/cvtt2
+ProdDir=${Root}/prod
+
+ReleaseHosts=("cloud21.cvtt.vpn")
+ReleasePorts=("22")
+ReleaseUsers=("cvttdist")
+ReleaseDir=("/home/cvttdist/software/cvtt2")
+# ----- Settings
+
+Project=${1}
+Version=${2}
+
+if [ "${Project}" == "" ]
+then
+    usage
+fi
+
+if [ "${Version}" == "" ]
+then
+    Version=latest
+fi
+
+
+function rsync_load_version() {
+    for idx in "${!ReleaseHosts[@]}"
+    do
+        host=${ReleaseHosts[${idx}]}
+        port=${ReleasePorts[${idx}]}
+        user=${ReleaseUsers[${idx}]}
+        rel_dir=${ReleaseDir[${idx}]}
+
+        if [ "${Version}" == "latest" ]; then
+            echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+            Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+            echo "Latest version is ${Version}"
+        fi
+        echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
+        if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
+        then
+            echo "Directory found..."
+            rsync_cmd="rsync -ahvv -e \"ssh -p ${port}\""
+            rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}"
+            rsync_cmd="${rsync_cmd} ${LocalSoftwareDir}/${Project}/"
+            echo ${rsync_cmd}
+            eval ${rsync_cmd}
+            status=$?
+            if [ ${status} -eq 0 ]
+            then
+                echo "Loading successful..."
+                break
+            fi
+        else
+            echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
+        fi
+    done
+    if [[ ! -d ${LocalSoftwareDir}/${Project} ]] ; then
+        echo ERROR loading software
+        exit 1
+    fi
+}
+
+mkdir -p ${LocalSoftwareDir}
+mkdir -p ${ProdDir}
+
+# exists and not empty
+rsync_load_version
+
+RelLocalSoftwareDir=../software/cvtt2
+Location="${RelLocalSoftwareDir}/${Project}/${Version}/${Project}"
+
+Cmd="cd ${ProdDir}"
+Cmd="${Cmd} && rm -rf ${Project}"
+Cmd="${Cmd} && ln -snf ${Location} ${Project}"
+
+echo ${Cmd} && eval ${Cmd}
+
+echo "Done: $0 $*"
+

pytorch/notebooks/Testing GPU.ipynb

@@ -1,8 +1,16 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "53fb9dcb-843b-4633-b031-22236bfeb815",
+   "metadata": {},
+   "source": [
+    "### Check CUDA device"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 8,
    "id": "6b269e64-be58-43b5-ad60-0fbd1d37861a",
    "metadata": {},
    "outputs": [
@@ -27,56 +35,34 @@
    ]
   },
   {
-   "cell_type": "code",
+   "cell_type": "markdown",
-   "execution_count": 2,
+   "id": "e6b4bc89-4cbd-4d4a-99a8-2d5bfb5d095d",
-   "id": "97906ea2-b284-4966-9c11-b8629f053815",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "import plotly"
+    "### Environment"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 10,
    "id": "7313a620-a0eb-4207-a12a-90aeee3cd980",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "'3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]'"
+       "('3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]',\n",
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import sys\n",
-    "sys.version"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "95d9a2e6-3464-4dbe-9a97-0c2d5eb34193",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
        " environ{'PATH': '/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',\n",
-       "        'HOSTNAME': '1fc69c311e22',\n",
+       "         'HOSTNAME': '6c24ffc52409',\n",
        "         'JUPYTER_ENABLE_LAB': 'yes',\n",
+       "         'PYTHONPATH': '/cvtt/prod',\n",
        "         'NVIDIA_VISIBLE_DEVICES': 'all',\n",
        "         'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility',\n",
        "         'LD_LIBRARY_PATH': '/usr/local/nvidia/lib:/usr/local/nvidia/lib64',\n",
        "         'PYTORCH_VERSION': '2.2.1',\n",
        "         'HOME': '/root',\n",
        "         'LC_CTYPE': 'C.UTF-8',\n",
-       "        'JPY_SESSION_NAME': '/workspace/oleg/Testing GPU.ipynb',\n",
+       "         'JPY_SESSION_NAME': '/workspace/Testing GPU.ipynb',\n",
        "         'JPY_PARENT_PID': '1',\n",
        "         'PYDEVD_USE_FRAME_EVAL': 'NO',\n",
        "         'TERM': 'xterm-color',\n",
@@ -86,31 +72,58 @@
        "         'PAGER': 'cat',\n",
        "         'GIT_PAGER': 'cat',\n",
        "         'MPLBACKEND': 'module://matplotlib_inline.backend_inline',\n",
-       "        'CUDA_MODULE_LOADING': 'LAZY'}"
+       "         'CUDA_MODULE_LOADING': 'LAZY'})"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "import os\n",
+    "import sys\n",
-    "import pandas as pd\n",
+    "sys.version,os.environ"
-    "import matplotlib\n",
+   ]
-    "os.environ"
+  },
+  {
+   "cell_type": "markdown",
+   "id": "943ac637-42c7-4a69-a6c4-94c382e22653",
+   "metadata": {},
+   "source": [
+    "### Test packages avai"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 11,
+   "id": "95d9a2e6-3464-4dbe-9a97-0c2d5eb34193",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "import matplotlib\n",
+    "import plotly\n",
+    "\n",
+    "import cvttpy.tools.timeutils as tu\n",
+    "\n",
+    "import autogluon\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
    "id": "eb38de31-fc19-4515-b08d-9cd7607ea958",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "75642bda0eed47598e340b8c1766949a",
+       "model_id": "2b2a98cdd61d477b811279a6753630a2",
        "version_major": 2,
        "version_minor": 0
       },
@@ -127,42 +140,37 @@
      "text": [
       "Done\n"
      ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "1717721060059418080"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "import ipywidgets\n",
     "from ipywidgets import interact, IntSlider\n",
+    "import cvttpy.tools.timeutils as tu\n",
     "\n",
     "def f(x):\n",
     "    return x\n",
     "\n",
     "interact(f, x=IntSlider(min=0, max=10, step=1, value=5))\n",
-    "print(\"Done\")\n"
+    "print(\"Done\")\n",
+    "tu.current_nanoseconds()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "f46e46a7-9b57-44aa-9bc9-dcbcf643bc88",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "jupyter-events==0.10.0\n",
-      "jupyter-lsp==2.2.5\n",
-      "jupyter_client==8.6.2\n",
-      "jupyter_core==5.7.2\n",
-      "jupyter_server==2.14.1\n",
-      "jupyter_server_terminals==0.5.3\n",
-      "jupyterlab==4.2.1\n",
-      "jupyterlab_pygments==0.3.0\n",
-      "jupyterlab_server==2.27.2\n",
-      "jupyterlab_widgets==3.0.11\n"
-     ]
-    }
-   ],
    "source": [
     "!pip freeze | grep jupyter"
    ]

tensorflow/notebooks/Testing GPU.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "id": "000d047d-3dfd-48cb-a3c3-bd1ad7c78b71",
    "metadata": {},
    "outputs": [
@@ -55,6 +55,14 @@
     "for gpu in gpus:\n",
     "    print(gpu)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "080fdf50-a5d5-4650-8f29-34c773443d59",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

tensorflow/notebooks/leo/LSTM_All_Crypto_01.ipynb

@@ -0,0 +1,443 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1023f2c1-e45f-4e1c-9a1b-66f59f128196",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Panda Version: 2.2.2\n",
+      "Today date is:  2024-06-06\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sqlite3\n",
+    "\n",
+    "import numpy as np\n",
+    "# It is apparently officially accepted to explicitly\n",
+    "# list all the functions you need from numpy:\n",
+    "from numpy import array, zeros, exp, random, dot\n",
+    "from numpy import shape, reshape, meshgrid, linspace\n",
+    "from numpy import hstack, vstack\n",
+    "\n",
+    "import pandas as pd\n",
+    "print('Panda Version:', pd.__version__)\n",
+    "\n",
+    "# Set ipython's max row display\n",
+    "pd.set_option('display.max_row', 100, 'display.max_columns', 25)\n",
+    "\n",
+    "import matplotlib.pyplot as plt # for plotting\n",
+    "import matplotlib\n",
+    "matplotlib.rcParams['figure.dpi'] = 100 # highres display\n",
+    "\n",
+    "import tensorflow as tf\n",
+    "from tensorflow import Variable\n",
+    "\n",
+    "from tensorflow.keras import Sequential\n",
+    "from tensorflow.keras import Model\n",
+    "from tensorflow.keras.layers import Dense\n",
+    "from tensorflow.keras.layers import Dropout\n",
+    "from tensorflow.keras.layers import TimeDistributed, RepeatVector\n",
+    "\n",
+    "from keras.optimizers import SGD\n",
+    "\n",
+    "from keras.models import load_model\n",
+    "from keras.callbacks import EarlyStopping\n",
+    "from keras.callbacks import ModelCheckpoint\n",
+    "\n",
+    "from keras.layers import LSTM, Dense, Concatenate\n",
+    "\n",
+    "from keras.optimizers import SGD\n",
+    "\n",
+    "import collections\n",
+    "from collections import Counter\n",
+    "\n",
+    "# Import date class from datetime module\n",
+    "import time\n",
+    "# import datetime as dt\n",
+    "from datetime import date, datetime\n",
+    "print(\"Today date is: \", date.today())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "c09a37a6-f0d9-48e3-a1d1-65ddaf2c489c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/workspace/leo\n",
+      "total 6528\n",
+      "drwxrwxr-x 1 oleg oleg     250 Jun  6 10:00 .\n",
+      "drwxrwxr-x 1 oleg oleg      18 Jun  3 23:40 ..\n",
+      "-rw------- 1 oleg oleg 1449984 Jun  4 00:49 20240601.mktdata.ohlcv.db\n",
+      "-rw------- 1 oleg oleg 1445888 Jun  3 23:44 20240602.mktdata.ohlcv.db\n",
+      "-rw------- 1 oleg oleg 1437696 Jun  4 16:45 20240603.mktdata.ohlcv.db\n",
+      "-rw------- 1 oleg oleg 1269760 Jun  5 10:00 20240604.mktdata.ohlcv.db\n",
+      "-rw------- 1 oleg oleg 1081344 Jun  6 10:00 20240605.mktdata.ohlcv.db\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pwd\n",
+    "\n",
+    "!ls -la /workspace/data/crypto_md/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d72d849-e8df-4564-a006-03ab646b9330",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "58d263ae-d028-4de0-9f10-0f14c4de28f7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mktdata_db_file = \"/workspace/data/crypto_md/20240601.mktdata.ohlcv.db\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70033f48-f283-43f3-8e86-3bd533a10fed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_conn = sqlite3.connect(mktdata_db_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "2cb82c91-a487-46b1-bcd0-012f7675010f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "    type                 name             tbl_name  rootpage  \\\n",
+      "0  table    bnbfut_ohlcv_1min    bnbfut_ohlcv_1min         2   \n",
+      "1  table   bnbspot_ohlcv_1min   bnbspot_ohlcv_1min        66   \n",
+      "2  table  coinbase_ohlcv_1min  coinbase_ohlcv_1min       224   \n",
+      "\n",
+      "                                                 sql  \n",
+      "0  CREATE TABLE bnbfut_ohlcv_1min (tstamp INTEGER...  \n",
+      "1  CREATE TABLE bnbspot_ohlcv_1min (tstamp INTEGE...  \n",
+      "2  CREATE TABLE coinbase_ohlcv_1min (tstamp INTEG...  \n"
+     ]
+    }
+   ],
+   "source": [
+    "tables_df = pd.read_sql_query(\"select * from sqlite_master where type = 'table'\", db_conn)\n",
+    "print (tables_df.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "cf159bde-1ccc-40e0-9eb9-a010c1372e07",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>tstamp</th>\n",
+       "      <th>exchange_id</th>\n",
+       "      <th>instrument_id</th>\n",
+       "      <th>open</th>\n",
+       "      <th>high</th>\n",
+       "      <th>low</th>\n",
+       "      <th>close</th>\n",
+       "      <th>volume</th>\n",
+       "      <th>vwap</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>5754</th>\n",
+       "      <td>1717286100000000000</td>\n",
+       "      <td>COINBASE</td>\n",
+       "      <td>PAIR-XRP-USD</td>\n",
+       "      <td>0.5183</td>\n",
+       "      <td>0.5183</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>7401.243996</td>\n",
+       "      <td>0.518096</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5755</th>\n",
+       "      <td>1717286160000000000</td>\n",
+       "      <td>COINBASE</td>\n",
+       "      <td>PAIR-XRP-USD</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>0.5182</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>3491.632415</td>\n",
+       "      <td>0.518020</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5756</th>\n",
+       "      <td>1717286220000000000</td>\n",
+       "      <td>COINBASE</td>\n",
+       "      <td>PAIR-XRP-USD</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>0.5180</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>6963.191620</td>\n",
+       "      <td>0.518095</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5757</th>\n",
+       "      <td>1717286280000000000</td>\n",
+       "      <td>COINBASE</td>\n",
+       "      <td>PAIR-XRP-USD</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>0.5183</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>0.5182</td>\n",
+       "      <td>35743.462893</td>\n",
+       "      <td>0.518140</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5758</th>\n",
+       "      <td>1717286340000000000</td>\n",
+       "      <td>COINBASE</td>\n",
+       "      <td>PAIR-XRP-USD</td>\n",
+       "      <td>0.5182</td>\n",
+       "      <td>0.5184</td>\n",
+       "      <td>0.5181</td>\n",
+       "      <td>0.5184</td>\n",
+       "      <td>45521.897366</td>\n",
+       "      <td>0.518245</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                   tstamp exchange_id instrument_id    open    high     low  \\\n",
+       "5754  1717286100000000000    COINBASE  PAIR-XRP-USD  0.5183  0.5183  0.5180   \n",
+       "5755  1717286160000000000    COINBASE  PAIR-XRP-USD  0.5180  0.5182  0.5180   \n",
+       "5756  1717286220000000000    COINBASE  PAIR-XRP-USD  0.5181  0.5181  0.5180   \n",
+       "5757  1717286280000000000    COINBASE  PAIR-XRP-USD  0.5181  0.5183  0.5181   \n",
+       "5758  1717286340000000000    COINBASE  PAIR-XRP-USD  0.5182  0.5184  0.5181   \n",
+       "\n",
+       "       close        volume      vwap  \n",
+       "5754  0.5180   7401.243996  0.518096  \n",
+       "5755  0.5180   3491.632415  0.518020  \n",
+       "5756  0.5181   6963.191620  0.518095  \n",
+       "5757  0.5182  35743.462893  0.518140  \n",
+       "5758  0.5184  45521.897366  0.518245  "
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)\n",
+    "df.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11d73b0a-fe9b-4d73-87c3-4250ef98b16a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 0601 - 0605"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd83027c-eaa1-44d9-98ad-8506099c323d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ccc60bc-6b74-408c-946c-bdbfee0cffb5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f358ee5-9ec0-4582-be09-4e4ad84faca7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fab70130-9661-4efd-ad45-7647d81cc6ce",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "77504183-34d8-42c6-93a3-d86a34388f9e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "                                  tstamp exchange_id instrument_id      open  \\\n",
+      "timestamp                                                                      \n",
+      "2024-06-01 00:00:00  1717200000000000000    COINBASE  PAIR-BTC-USD  67473.07   \n",
+      "2024-06-01 00:01:00  1717200060000000000    COINBASE  PAIR-BTC-USD  67506.57   \n",
+      "2024-06-01 00:02:00  1717200120000000000    COINBASE  PAIR-BTC-USD  67494.73   \n",
+      "2024-06-01 00:03:00  1717200180000000000    COINBASE  PAIR-BTC-USD  67512.78   \n",
+      "2024-06-01 00:04:00  1717200240000000000    COINBASE  PAIR-BTC-USD  67538.81   \n",
+      "\n",
+      "                         high       low     close    volume          vwap  \n",
+      "timestamp                                                                  \n",
+      "2024-06-01 00:00:00  67514.99  67468.13  67506.03  2.045049  67496.869352  \n",
+      "2024-06-01 00:01:00  67515.00  67480.66  67494.74  0.646759  67500.893305  \n",
+      "2024-06-01 00:02:00  67515.00  67455.91  67514.99  9.732906  67477.840770  \n",
+      "2024-06-01 00:03:00  67553.85  67496.58  67539.76  4.374730  67522.912057  \n",
+      "2024-06-01 00:04:00  67579.13  67530.05  67579.13  2.215383  67557.086208  \n"
+     ]
+    }
+   ],
+   "source": [
+    "df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n",
+    "df.set_index(\"timestamp\", inplace=True)\n",
+    "print (df.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "316c0cfb-c73d-4dad-9d49-eb80daa229ec",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "92700c76-8eac-4ebb-86d3-27066486c437",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3604f3f2-3e56-477d-a7ac-d6490839642d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_union = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a7c8b332-cd4a-455f-b7cf-381aec15c456",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a27972a-f457-4ca5-8530-d6c87c7d9d91",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b5356525-e614-4858-af4e-648c03b7d21e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "695de24a-41f4-4c82-a0dd-dbbd7bfba2bd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c6d80b3-4a46-432b-9b88-6a264f9b7a7e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0rc1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}