From 1013e80286ab9ddde0e65d70099e2f3eccb2ef90 Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Tue, 4 Jun 2024 00:56:03 +0000 Subject: [PATCH] progress: added tensorflow container and shared data directory --- load_market_data.sh | 57 + tensorflow/.env | 2 + tensorflow/Dockerfile | 31 + tensorflow/docker-compose.yml | 25 + .../Loading Market Data Example.ipynb | 1221 +++++++++++++++++ tensorflow/notebooks/Testing GPU.ipynb | 75 + 6 files changed, 1411 insertions(+) create mode 100755 load_market_data.sh create mode 100644 tensorflow/.env create mode 100644 tensorflow/Dockerfile create mode 100644 tensorflow/docker-compose.yml create mode 100644 tensorflow/notebooks/Loading Market Data Example.ipynb create mode 100644 tensorflow/notebooks/Testing GPU.ipynb diff --git a/load_market_data.sh b/load_market_data.sh new file mode 100755 index 0000000..74ff49b --- /dev/null +++ b/load_market_data.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +date=${1} +if [ "${date}" == "" ] ; then + date="yesterday" +fi + +SourceHost=cryptoval3.cvtt.vpn +SourceUser=cvtt +SourceRootDir=/localdisk/cvtt/archive/md_archive +SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db") +SourceFileZip="${SourceFile}.gz" +SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}") + +TargetDir="/opt/jupyter_gpu/data/crypto_md" +TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db") +TargetFilePath="${TargetDir}/${TargetFile}" + +echo ${SourceFile} +tmp_dir=$(mktemp -d) + +function cleanup { + cd ${HOME} + rm -rf ${tmp_dir} +} +trap cleanup EXIT + +function download_file { + + Cmd="rsync" + Cmd="${Cmd} -ahv" + if tty -s; then + Cmd="${Cmd} --progress=info2" + fi + Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/" + echo ${Cmd} + eval ${Cmd} + ls -lh ${tmp_dir} + Cmd="gunzip ${tmp_dir}/${SourceFileZip}" + echo ${Cmd} && eval ${Cmd} + ls -lh ${tmp_dir} + + rm -f ${TargetFilePath} + touch ${TargetFilePath} + + for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min + do + Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}" + echo ${Cmd} + eval ${Cmd} + done + chmod 600 ${TargetFilePath} + ls -lh ${TargetFilePath} + + +} +download_file diff --git a/tensorflow/.env b/tensorflow/.env new file mode 100644 index 0000000..267b92a --- /dev/null +++ b/tensorflow/.env @@ -0,0 +1,2 @@ +ROOT_DIR=/opt/jupyter_gpu/tensorflow +JUPYTER_PORT=9999 diff --git a/tensorflow/Dockerfile b/tensorflow/Dockerfile new file mode 100644 index 0000000..97776b9 --- /dev/null +++ b/tensorflow/Dockerfile @@ -0,0 +1,31 @@ +# Use the official TensorFlow image with GPU support +FROM tensorflow/tensorflow:latest-gpu-jupyter + +# Install necessary packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -s /bin/bash oleg + +RUN mkdir -p /workspace/tf_cache + +# Install packages +RUN pip install tensorflow-gpu +RUN pip install jupyterlab +RUN pip install plotly + +# Install additional applications +RUN apt update +RUN apt-get install graphviz -y + +# Set the working directory +WORKDIR /workspace + +# Expose the port for JupyterLab +EXPOSE ${JUPYTER_PORT} + +# Command to run JupyterLab +CMD ["jupyter-lab", "--ip=0.0.0.0", "--port=9999", "--no-browser", "--allow-root", "--NotebookApp.token=''"] diff --git a/tensorflow/docker-compose.yml b/tensorflow/docker-compose.yml new file mode 100644 index 0000000..bc67f84 --- /dev/null +++ b/tensorflow/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.8' + +#================================================== +# Relies on the file `.env` content for varables: +# ROOT_DIR +# JUPYTER_PORT +#================================================== + +services: + jupyter: + user: "oleg" + build: ${ROOT_DIR} + image: cvtt/jupyter_gpu_tf:v1.0.2 + container_name: jupyter_gpu_tf + runtime: nvidia + environment: + - JUPYTER_ENABLE_LAB=yes + - TF_CACHE_DIR=/workspace/tf_cache + volumes: + - ${ROOT_DIR}/notebooks:/workspace + - ${ROOT_DIR}/jupyter_settings/user-settings:/home/oleg/.jupyter/lab/user-settings + - ${ROOT_DIR}/jupyter_settings/workspaces:/home/oleg/.jupyter/lab/workspaces + ports: + - "${JUPYTER_PORT}:${JUPYTER_PORT}" + shm_size: "8gb" diff --git a/tensorflow/notebooks/Loading Market Data Example.ipynb b/tensorflow/notebooks/Loading Market Data Example.ipynb new file mode 100644 index 0000000..10b7165 --- /dev/null +++ b/tensorflow/notebooks/Loading Market Data Example.ipynb @@ -0,0 +1,1221 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "89251b34-9738-4fa3-b2db-7d6a7dde7d48", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "import pandas as pd\n", + "from datetime import datetime\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "0ecd2b95-b212-44a4-ac15-ff7aa4f032f9", + "metadata": {}, + "source": [ + "### Here you the database file containing 1 day of 1min data\n", + "The name of the file contains the date." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eb24b805-ee73-4db4-b1de-0c43bea7d86f", + "metadata": {}, + "outputs": [], + "source": [ + "mktdata_db_file = \"/workspace/data/crypto_md/20240602.mktdata.ohlcv.db\"" + ] + }, + { + "cell_type": "markdown", + "id": "b2b7808c-1c78-49cb-9081-bde325b4f30a", + "metadata": {}, + "source": [ + "### Simple query that gets everything into DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "8dcb0ec1-4e55-4d13-91fc-57c24fac6460", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
01717286400000000000COINBASEPAIR-BTC-USD67719.290067721.730067708.110067719.96001.88824067713.845756
11717286460000000000COINBASEPAIR-BTC-USD67714.160067717.350067707.680067709.88000.65535867712.153017
21717286520000000000COINBASEPAIR-BTC-USD67709.880067709.880067689.700067691.31001.00150667697.965852
31717286580000000000COINBASEPAIR-BTC-USD67692.330067713.060067691.220067713.06005.65254767695.441479
41717286640000000000COINBASEPAIR-BTC-USD67713.060067721.330067710.770067715.05002.99752567715.683416
..............................
57551717372500000000000COINBASEPAIR-XRP-USD0.51350.51360.51340.51362505.0220900.513596
57561717372560000000000COINBASEPAIR-XRP-USD0.51350.51350.51340.51345612.8217630.513489
57571717372620000000000COINBASEPAIR-XRP-USD0.51340.51350.51330.51335503.4520730.513377
57581717372680000000000COINBASEPAIR-XRP-USD0.51330.51350.51330.51343925.9795990.513428
57591717372740000000000COINBASEPAIR-XRP-USD0.51350.51350.51330.513321244.6698390.513309
\n", + "

5760 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " tstamp exchange_id instrument_id open high \\\n", + "0 1717286400000000000 COINBASE PAIR-BTC-USD 67719.2900 67721.7300 \n", + "1 1717286460000000000 COINBASE PAIR-BTC-USD 67714.1600 67717.3500 \n", + "2 1717286520000000000 COINBASE PAIR-BTC-USD 67709.8800 67709.8800 \n", + "3 1717286580000000000 COINBASE PAIR-BTC-USD 67692.3300 67713.0600 \n", + "4 1717286640000000000 COINBASE PAIR-BTC-USD 67713.0600 67721.3300 \n", + "... ... ... ... ... ... \n", + "5755 1717372500000000000 COINBASE PAIR-XRP-USD 0.5135 0.5136 \n", + "5756 1717372560000000000 COINBASE PAIR-XRP-USD 0.5135 0.5135 \n", + "5757 1717372620000000000 COINBASE PAIR-XRP-USD 0.5134 0.5135 \n", + "5758 1717372680000000000 COINBASE PAIR-XRP-USD 0.5133 0.5135 \n", + "5759 1717372740000000000 COINBASE PAIR-XRP-USD 0.5135 0.5135 \n", + "\n", + " low close volume vwap \n", + "0 67708.1100 67719.9600 1.888240 67713.845756 \n", + "1 67707.6800 67709.8800 0.655358 67712.153017 \n", + "2 67689.7000 67691.3100 1.001506 67697.965852 \n", + "3 67691.2200 67713.0600 5.652547 67695.441479 \n", + "4 67710.7700 67715.0500 2.997525 67715.683416 \n", + "... ... ... ... ... \n", + "5755 0.5134 0.5136 2505.022090 0.513596 \n", + "5756 0.5134 0.5134 5612.821763 0.513489 \n", + "5757 0.5133 0.5133 5503.452073 0.513377 \n", + "5758 0.5133 0.5134 3925.979599 0.513428 \n", + "5759 0.5133 0.5133 21244.669839 0.513309 \n", + "\n", + "[5760 rows x 9 columns]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db_conn = sqlite3.connect(mktdata_db_file)\n", + "\n", + "df = pd.read_sql_query(\"select * from coinbase_ohlcv_1min\", db_conn)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "a01c3aad-7e84-41ce-a835-06f35bd65668", + "metadata": {}, + "source": [ + "### Timestamp is not very convenient, so we can make it look nicer" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c00fce97-69d0-47d2-be97-410d8c826292", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
timestamp
2024-06-02 00:00:001717286400000000000COINBASEPAIR-BTC-USD67719.290067721.730067708.110067719.96001.88824067713.845756
2024-06-02 00:01:001717286460000000000COINBASEPAIR-BTC-USD67714.160067717.350067707.680067709.88000.65535867712.153017
2024-06-02 00:02:001717286520000000000COINBASEPAIR-BTC-USD67709.880067709.880067689.700067691.31001.00150667697.965852
2024-06-02 00:03:001717286580000000000COINBASEPAIR-BTC-USD67692.330067713.060067691.220067713.06005.65254767695.441479
2024-06-02 00:04:001717286640000000000COINBASEPAIR-BTC-USD67713.060067721.330067710.770067715.05002.99752567715.683416
..............................
2024-06-02 23:55:001717372500000000000COINBASEPAIR-XRP-USD0.51350.51360.51340.51362505.0220900.513596
2024-06-02 23:56:001717372560000000000COINBASEPAIR-XRP-USD0.51350.51350.51340.51345612.8217630.513489
2024-06-02 23:57:001717372620000000000COINBASEPAIR-XRP-USD0.51340.51350.51330.51335503.4520730.513377
2024-06-02 23:58:001717372680000000000COINBASEPAIR-XRP-USD0.51330.51350.51330.51343925.9795990.513428
2024-06-02 23:59:001717372740000000000COINBASEPAIR-XRP-USD0.51350.51350.51330.513321244.6698390.513309
\n", + "

5760 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " tstamp exchange_id instrument_id \\\n", + "timestamp \n", + "2024-06-02 00:00:00 1717286400000000000 COINBASE PAIR-BTC-USD \n", + "2024-06-02 00:01:00 1717286460000000000 COINBASE PAIR-BTC-USD \n", + "2024-06-02 00:02:00 1717286520000000000 COINBASE PAIR-BTC-USD \n", + "2024-06-02 00:03:00 1717286580000000000 COINBASE PAIR-BTC-USD \n", + "2024-06-02 00:04:00 1717286640000000000 COINBASE PAIR-BTC-USD \n", + "... ... ... ... \n", + "2024-06-02 23:55:00 1717372500000000000 COINBASE PAIR-XRP-USD \n", + "2024-06-02 23:56:00 1717372560000000000 COINBASE PAIR-XRP-USD \n", + "2024-06-02 23:57:00 1717372620000000000 COINBASE PAIR-XRP-USD \n", + "2024-06-02 23:58:00 1717372680000000000 COINBASE PAIR-XRP-USD \n", + "2024-06-02 23:59:00 1717372740000000000 COINBASE PAIR-XRP-USD \n", + "\n", + " open high low close \\\n", + "timestamp \n", + "2024-06-02 00:00:00 67719.2900 67721.7300 67708.1100 67719.9600 \n", + "2024-06-02 00:01:00 67714.1600 67717.3500 67707.6800 67709.8800 \n", + "2024-06-02 00:02:00 67709.8800 67709.8800 67689.7000 67691.3100 \n", + "2024-06-02 00:03:00 67692.3300 67713.0600 67691.2200 67713.0600 \n", + "2024-06-02 00:04:00 67713.0600 67721.3300 67710.7700 67715.0500 \n", + "... ... ... ... ... \n", + "2024-06-02 23:55:00 0.5135 0.5136 0.5134 0.5136 \n", + "2024-06-02 23:56:00 0.5135 0.5135 0.5134 0.5134 \n", + "2024-06-02 23:57:00 0.5134 0.5135 0.5133 0.5133 \n", + "2024-06-02 23:58:00 0.5133 0.5135 0.5133 0.5134 \n", + "2024-06-02 23:59:00 0.5135 0.5135 0.5133 0.5133 \n", + "\n", + " volume vwap \n", + "timestamp \n", + "2024-06-02 00:00:00 1.888240 67713.845756 \n", + "2024-06-02 00:01:00 0.655358 67712.153017 \n", + "2024-06-02 00:02:00 1.001506 67697.965852 \n", + "2024-06-02 00:03:00 5.652547 67695.441479 \n", + "2024-06-02 00:04:00 2.997525 67715.683416 \n", + "... ... ... \n", + "2024-06-02 23:55:00 2505.022090 0.513596 \n", + "2024-06-02 23:56:00 5612.821763 0.513489 \n", + "2024-06-02 23:57:00 5503.452073 0.513377 \n", + "2024-06-02 23:58:00 3925.979599 0.513428 \n", + "2024-06-02 23:59:00 21244.669839 0.513309 \n", + "\n", + "[5760 rows x 9 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n", + "df.set_index(\"timestamp\", inplace=True)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "62b24872-8425-44d0-b37a-29792d88f20f", + "metadata": {}, + "source": [ + "### If you want individual instruments, you could create the function" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "75929164-87f7-4125-b20b-5de745b9b9c4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
timestamp
2024-06-02 00:00:001717286400000000000COINBASEPAIR-BTC-USD67719.2967721.7367708.1167719.961.88824067713.845756
2024-06-02 00:01:001717286460000000000COINBASEPAIR-BTC-USD67714.1667717.3567707.6867709.880.65535867712.153017
2024-06-02 00:02:001717286520000000000COINBASEPAIR-BTC-USD67709.8867709.8867689.7067691.311.00150667697.965852
2024-06-02 00:03:001717286580000000000COINBASEPAIR-BTC-USD67692.3367713.0667691.2267713.065.65254767695.441479
2024-06-02 00:04:001717286640000000000COINBASEPAIR-BTC-USD67713.0667721.3367710.7767715.052.99752567715.683416
..............................
2024-06-02 23:55:001717372500000000000COINBASEPAIR-BTC-USD67757.0967757.2667750.3267753.930.32611767753.207138
2024-06-02 23:56:001717372560000000000COINBASEPAIR-BTC-USD67755.0067761.3467735.0567755.351.25945767744.858199
2024-06-02 23:57:001717372620000000000COINBASEPAIR-BTC-USD67755.3967759.8867753.7467757.490.42957867757.147946
2024-06-02 23:58:001717372680000000000COINBASEPAIR-BTC-USD67757.2767759.9067748.4167750.710.68367867756.292991
2024-06-02 23:59:001717372740000000000COINBASEPAIR-BTC-USD67752.1467754.5067725.2567735.520.53094367747.261494
\n", + "

1440 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " tstamp exchange_id instrument_id open \\\n", + "timestamp \n", + "2024-06-02 00:00:00 1717286400000000000 COINBASE PAIR-BTC-USD 67719.29 \n", + "2024-06-02 00:01:00 1717286460000000000 COINBASE PAIR-BTC-USD 67714.16 \n", + "2024-06-02 00:02:00 1717286520000000000 COINBASE PAIR-BTC-USD 67709.88 \n", + "2024-06-02 00:03:00 1717286580000000000 COINBASE PAIR-BTC-USD 67692.33 \n", + "2024-06-02 00:04:00 1717286640000000000 COINBASE PAIR-BTC-USD 67713.06 \n", + "... ... ... ... ... \n", + "2024-06-02 23:55:00 1717372500000000000 COINBASE PAIR-BTC-USD 67757.09 \n", + "2024-06-02 23:56:00 1717372560000000000 COINBASE PAIR-BTC-USD 67755.00 \n", + "2024-06-02 23:57:00 1717372620000000000 COINBASE PAIR-BTC-USD 67755.39 \n", + "2024-06-02 23:58:00 1717372680000000000 COINBASE PAIR-BTC-USD 67757.27 \n", + "2024-06-02 23:59:00 1717372740000000000 COINBASE PAIR-BTC-USD 67752.14 \n", + "\n", + " high low close volume vwap \n", + "timestamp \n", + "2024-06-02 00:00:00 67721.73 67708.11 67719.96 1.888240 67713.845756 \n", + "2024-06-02 00:01:00 67717.35 67707.68 67709.88 0.655358 67712.153017 \n", + "2024-06-02 00:02:00 67709.88 67689.70 67691.31 1.001506 67697.965852 \n", + "2024-06-02 00:03:00 67713.06 67691.22 67713.06 5.652547 67695.441479 \n", + "2024-06-02 00:04:00 67721.33 67710.77 67715.05 2.997525 67715.683416 \n", + "... ... ... ... ... ... \n", + "2024-06-02 23:55:00 67757.26 67750.32 67753.93 0.326117 67753.207138 \n", + "2024-06-02 23:56:00 67761.34 67735.05 67755.35 1.259457 67744.858199 \n", + "2024-06-02 23:57:00 67759.88 67753.74 67757.49 0.429578 67757.147946 \n", + "2024-06-02 23:58:00 67759.90 67748.41 67750.71 0.683678 67756.292991 \n", + "2024-06-02 23:59:00 67754.50 67725.25 67735.52 0.530943 67747.261494 \n", + "\n", + "[1440 rows x 9 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
timestamp
2024-06-02 00:00:001717286400000000000COINBASEPAIR-ETH-USD3812.493812.923812.483812.65296.9842833812.885799
2024-06-02 00:01:001717286460000000000COINBASEPAIR-ETH-USD3812.883812.923812.663812.9222.5847593812.885708
2024-06-02 00:02:001717286520000000000COINBASEPAIR-ETH-USD3812.923812.923812.173812.1736.2307013812.738323
2024-06-02 00:03:001717286580000000000COINBASEPAIR-ETH-USD3812.163813.173812.143813.1766.9844053812.827731
2024-06-02 00:04:001717286640000000000COINBASEPAIR-ETH-USD3813.163813.173813.133813.1782.7813253813.169087
..............................
2024-06-02 23:55:001717372500000000000COINBASEPAIR-ETH-USD3783.343784.433783.333783.5717.4093283783.902157
2024-06-02 23:56:001717372560000000000COINBASEPAIR-ETH-USD3783.573783.863782.203783.866.6328753783.094935
2024-06-02 23:57:001717372620000000000COINBASEPAIR-ETH-USD3783.863783.943782.163782.187.8262353782.958328
2024-06-02 23:58:001717372680000000000COINBASEPAIR-ETH-USD3782.233782.243779.833779.869.0204923781.022292
2024-06-02 23:59:001717372740000000000COINBASEPAIR-ETH-USD3779.863780.053778.263778.8839.5888103778.964208
\n", + "

1440 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " tstamp exchange_id instrument_id open \\\n", + "timestamp \n", + "2024-06-02 00:00:00 1717286400000000000 COINBASE PAIR-ETH-USD 3812.49 \n", + "2024-06-02 00:01:00 1717286460000000000 COINBASE PAIR-ETH-USD 3812.88 \n", + "2024-06-02 00:02:00 1717286520000000000 COINBASE PAIR-ETH-USD 3812.92 \n", + "2024-06-02 00:03:00 1717286580000000000 COINBASE PAIR-ETH-USD 3812.16 \n", + "2024-06-02 00:04:00 1717286640000000000 COINBASE PAIR-ETH-USD 3813.16 \n", + "... ... ... ... ... \n", + "2024-06-02 23:55:00 1717372500000000000 COINBASE PAIR-ETH-USD 3783.34 \n", + "2024-06-02 23:56:00 1717372560000000000 COINBASE PAIR-ETH-USD 3783.57 \n", + "2024-06-02 23:57:00 1717372620000000000 COINBASE PAIR-ETH-USD 3783.86 \n", + "2024-06-02 23:58:00 1717372680000000000 COINBASE PAIR-ETH-USD 3782.23 \n", + "2024-06-02 23:59:00 1717372740000000000 COINBASE PAIR-ETH-USD 3779.86 \n", + "\n", + " high low close volume vwap \n", + "timestamp \n", + "2024-06-02 00:00:00 3812.92 3812.48 3812.65 296.984283 3812.885799 \n", + "2024-06-02 00:01:00 3812.92 3812.66 3812.92 22.584759 3812.885708 \n", + "2024-06-02 00:02:00 3812.92 3812.17 3812.17 36.230701 3812.738323 \n", + "2024-06-02 00:03:00 3813.17 3812.14 3813.17 66.984405 3812.827731 \n", + "2024-06-02 00:04:00 3813.17 3813.13 3813.17 82.781325 3813.169087 \n", + "... ... ... ... ... ... \n", + "2024-06-02 23:55:00 3784.43 3783.33 3783.57 17.409328 3783.902157 \n", + "2024-06-02 23:56:00 3783.86 3782.20 3783.86 6.632875 3783.094935 \n", + "2024-06-02 23:57:00 3783.94 3782.16 3782.18 7.826235 3782.958328 \n", + "2024-06-02 23:58:00 3782.24 3779.83 3779.86 9.020492 3781.022292 \n", + "2024-06-02 23:59:00 3780.05 3778.26 3778.88 39.588810 3778.964208 \n", + "\n", + "[1440 rows x 9 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tstampexchange_idinstrument_idopenhighlowclosevolumevwap
timestamp
2024-06-02 00:00:001717286400000000000COINBASEPAIR-XRP-USD0.51840.51840.51800.518168137.1589440.518082
2024-06-02 00:01:001717286460000000000COINBASEPAIR-XRP-USD0.51800.51820.51790.51823659.6098830.518028
2024-06-02 00:02:001717286520000000000COINBASEPAIR-XRP-USD0.51820.51820.51790.51809693.3791220.517920
2024-06-02 00:03:001717286580000000000COINBASEPAIR-XRP-USD0.51810.51840.51800.51848324.1078320.518215
2024-06-02 00:04:001717286640000000000COINBASEPAIR-XRP-USD0.51840.51840.51830.51831482.0655650.518348
..............................
2024-06-02 23:55:001717372500000000000COINBASEPAIR-XRP-USD0.51350.51360.51340.51362505.0220900.513596
2024-06-02 23:56:001717372560000000000COINBASEPAIR-XRP-USD0.51350.51350.51340.51345612.8217630.513489
2024-06-02 23:57:001717372620000000000COINBASEPAIR-XRP-USD0.51340.51350.51330.51335503.4520730.513377
2024-06-02 23:58:001717372680000000000COINBASEPAIR-XRP-USD0.51330.51350.51330.51343925.9795990.513428
2024-06-02 23:59:001717372740000000000COINBASEPAIR-XRP-USD0.51350.51350.51330.513321244.6698390.513309
\n", + "

1440 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " tstamp exchange_id instrument_id open \\\n", + "timestamp \n", + "2024-06-02 00:00:00 1717286400000000000 COINBASE PAIR-XRP-USD 0.5184 \n", + "2024-06-02 00:01:00 1717286460000000000 COINBASE PAIR-XRP-USD 0.5180 \n", + "2024-06-02 00:02:00 1717286520000000000 COINBASE PAIR-XRP-USD 0.5182 \n", + "2024-06-02 00:03:00 1717286580000000000 COINBASE PAIR-XRP-USD 0.5181 \n", + "2024-06-02 00:04:00 1717286640000000000 COINBASE PAIR-XRP-USD 0.5184 \n", + "... ... ... ... ... \n", + "2024-06-02 23:55:00 1717372500000000000 COINBASE PAIR-XRP-USD 0.5135 \n", + "2024-06-02 23:56:00 1717372560000000000 COINBASE PAIR-XRP-USD 0.5135 \n", + "2024-06-02 23:57:00 1717372620000000000 COINBASE PAIR-XRP-USD 0.5134 \n", + "2024-06-02 23:58:00 1717372680000000000 COINBASE PAIR-XRP-USD 0.5133 \n", + "2024-06-02 23:59:00 1717372740000000000 COINBASE PAIR-XRP-USD 0.5135 \n", + "\n", + " high low close volume vwap \n", + "timestamp \n", + "2024-06-02 00:00:00 0.5184 0.5180 0.5181 68137.158944 0.518082 \n", + "2024-06-02 00:01:00 0.5182 0.5179 0.5182 3659.609883 0.518028 \n", + "2024-06-02 00:02:00 0.5182 0.5179 0.5180 9693.379122 0.517920 \n", + "2024-06-02 00:03:00 0.5184 0.5180 0.5184 8324.107832 0.518215 \n", + "2024-06-02 00:04:00 0.5184 0.5183 0.5183 1482.065565 0.518348 \n", + "... ... ... ... ... ... \n", + "2024-06-02 23:55:00 0.5136 0.5134 0.5136 2505.022090 0.513596 \n", + "2024-06-02 23:56:00 0.5135 0.5134 0.5134 5612.821763 0.513489 \n", + "2024-06-02 23:57:00 0.5135 0.5133 0.5133 5503.452073 0.513377 \n", + "2024-06-02 23:58:00 0.5135 0.5133 0.5134 3925.979599 0.513428 \n", + "2024-06-02 23:59:00 0.5135 0.5133 0.5133 21244.669839 0.513309 \n", + "\n", + "[1440 rows x 9 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_instrument_md(instrument_id: str) -> pd.DataFrame:\n", + " query = f\"select * from coinbase_ohlcv_1min where instrument_id = '{instrument_id}'\"\n", + " df = pd.read_sql_query(query, db_conn)\n", + " df[\"timestamp\"] = pd.to_datetime(df['tstamp'], unit='ns')\n", + " df.set_index(\"timestamp\", inplace=True)\n", + " return df\n", + "\n", + "btc_df = get_instrument_md(instrument_id=\"PAIR-BTC-USD\")\n", + "eth_df = get_instrument_md(instrument_id=\"PAIR-ETH-USD\")\n", + "xrp_df = get_instrument_md(instrument_id=\"PAIR-XRP-USD\")\n", + "\n", + "from IPython.display import display\n", + "display(btc_df)\n", + "display(eth_df)\n", + "display(xrp_df)\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0rc1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tensorflow/notebooks/Testing GPU.ipynb b/tensorflow/notebooks/Testing GPU.ipynb new file mode 100644 index 0000000..66f48e9 --- /dev/null +++ b/tensorflow/notebooks/Testing GPU.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "000d047d-3dfd-48cb-a3c3-bd1ad7c78b71", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TensorFlow version: 2.16.1\n", + "GPUs detected: 1\n", + " - /physical_device:GPU:0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-06-03 21:52:43.785816: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2024-06-03 21:52:43.795907: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2024-06-03 21:52:43.799970: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "\n", + "def get_available_gpus():\n", + " gpus = tf.config.list_physical_devices('GPU')\n", + " if gpus:\n", + " print(f\"GPUs detected: {len(gpus)}\")\n", + " for gpu in gpus:\n", + " print(f\" - {gpu.name}\")\n", + " else:\n", + " print(\"No GPUs detected.\")\n", + "\n", + "if __name__ == \"__main__\":\n", + " print(\"TensorFlow version:\", tf.__version__)\n", + " get_available_gpus()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "324dc7a1-61b8-4384-bfed-4385e7dbe49d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0rc1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}