progress

load_market_data.sh

@@ -5,9 +5,11 @@ if [ "${date}" == "" ] ; then
     date="yesterday"
 fi
 
-SourceHost=cryptoval3.cvtt.vpn
+SourceHost=cloud21.cvtt.vpn
 SourceUser=cvtt
-SourceRootDir=/localdisk/cvtt/archive/md_archive
+SourceRootDir=/opt/store/cvtt/md_archive/crypto/cloud21
+#SourceRootDir=/opt/store/cvtt/md_archive/crypto/cvttdata
+
 SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
 SourceFileZip="${SourceFile}.gz"
 SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")

pytorch/.env

@@ -1,5 +1,5 @@
 ROOT_DIR=/opt/jupyter_gpu/pytorch
 DATA_DIR=/opt/jupyter_gpu/data
 JUPYTER_PORT=8888
-PUID=1000
+# PUID=1000
-PGID=1000
+# PGID=1000

pytorch/Dockerfile

@@ -18,6 +18,7 @@ RUN apt-get install graphviz -y
 
 # Set environment variables, etc.
 #ENV MY_ENV_VAR=myvalue
+COPY cvttpy /cvttpy
 
 # Set the working directory
 WORKDIR /workspace

pytorch/docker-compose.yml

@@ -10,23 +10,19 @@ version: '3.8'
 #==================================================
 services:
   jupyter:
-    user: "${UID}:${GID}"
+    # user: "${UID}:${GID}"
-    build: ${ROOT_DIR}
+    image: cloud21.cvtt.vpn:5500/jupyter_gpu_pytorch:latest
-    image: cvtt/jupyter_gpu_pt:v1.1.5
     container_name: jupyter_gpu_pt
     runtime: nvidia
     restart: unless-stopped
     environment:
       - JUPYTER_ENABLE_LAB=yes
       - PYTHONPATH=/cvtt/prod
-#    security_opt:
-#      - no-new-privileges:true
     volumes:
       - ${ROOT_DIR}/notebooks:/workspace
       - ${ROOT_DIR}/jupyter_settings/user-settings:/root/.jupyter/lab/user-settings
       - ${ROOT_DIR}/jupyter_settings/workspaces:/root/.jupyter/lab/workspaces
       - ${ROOT_DIR}/.cache/torch:/root/.cache/torch
-      - ${ROOT_DIR}/cvtt:/cvtt
 
       - ${DATA_DIR}:/workspace/data
     ports:

pytorch/notebooks/Testing GPU.ipynb

@@ -246,7 +246,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

tensorflow/Dockerfile

@@ -20,6 +20,13 @@ RUN pip install plotly
 RUN pip install seaborn
 RUN pip install statsmodels
 
+RUN pip install scikit-learn
+RUN pip install Arch
+RUN pip install Quantlib
+RUN pip install Backtrader
+RUN pip install pyfolio
+RUN pip install XGBoost
+
 # Install additional applications
 RUN apt update
 RUN apt-get install graphviz -y

tensorflow/notebooks/Loading Market Data Example.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "89251b34-9738-4fa3-b2db-7d6a7dde7d48",
    "metadata": {},
    "outputs": [],
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "eb24b805-ee73-4db4-b1de-0c43bea7d86f",
    "metadata": {},
    "outputs": [],
@@ -42,7 +42,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 5,
    "id": "8dcb0ec1-4e55-4d13-91fc-57c24fac6460",
    "metadata": {},
    "outputs": [
@@ -246,7 +246,7 @@
        "[5760 rows x 9 columns]"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }

tensorflow/notebooks/leo/LSTM_All_Crypto_01.ipynb

@@ -210,25 +210,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 179,
+   "execution_count": 2,
    "id": "316c0cfb-c73d-4dad-9d49-eb80daa229ec",
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "ename": "NameError",
-     "output_type": "stream",
+     "evalue": "name 'df_concat' is not defined",
-     "text": [
+     "output_type": "error",
-      "========================================\n",
+     "traceback": [
-      "     date  close_xrp    volume_xrp  vwap_xrp  close_eth  volume_eth  vwap_eth  \\\n",
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "0   1-0-0     0.5173  28809.165339  0.517217   0.376204   37.497964  0.376108   \n",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "1   1-0-1     0.5170  10184.511212  0.517060   0.375942   11.703305  0.376013   \n",
+      "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# BTC\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m df_concat_btc \u001b[38;5;241m=\u001b[39m \u001b[43mdf_concat\u001b[49m[df_concat[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minstrument_id\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPAIR-BTC-USD\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# Reset Index\u001b[39;00m\n\u001b[1;32m      5\u001b[0m df_concat_btc \u001b[38;5;241m=\u001b[39m df_concat_btc\u001b[38;5;241m.\u001b[39mreset_index(drop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m)\n",
-      "2  1-0-10     0.5176  15515.348778  0.517601   0.375938   11.922897  0.375935   \n",
+      "\u001b[0;31mNameError\u001b[0m: name 'df_concat' is not defined"
-      "\n",
-      "   close_ltc  volume_ltc  vwap_ltc  close_btc  volume_btc  vwap_btc  \n",
-      "0     0.8317  110.056253  0.831662   0.675060    2.045049  0.674969  \n",
-      "1     0.8312  176.868598  0.831441   0.674947    0.646759  0.675009  \n",
-      "2     0.8311   58.951033  0.831037   0.675507    1.201335  0.675478  \n",
-      "test sample size:  16698\n"
      ]
     }
    ],
@@ -323,6 +317,36 @@
     "print (\"test sample size: \", sample_size)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b193e3d7-7907-4137-a63f-4695f76aae21",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'sample' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m (\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msample.shape[0] shape:  \u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[43msample\u001b[49m\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m])\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'sample' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "print ('sample.shape[0] shape:  ', sample.shape[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "982cc6d4-28f7-4338-b51d-17666c90541c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 177,