progress: added tensorflow container and shared data directory

load_market_data.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+date=${1}
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+SourceHost=cryptoval3.cvtt.vpn
+SourceUser=cvtt
+SourceRootDir=/localdisk/cvtt/archive/md_archive
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then    
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file

tensorflow/.env

@@ -0,0 +1,2 @@
+ROOT_DIR=/opt/jupyter_gpu/tensorflow
+JUPYTER_PORT=9999

tensorflow/Dockerfile

@@ -0,0 +1,31 @@
+# Use the official TensorFlow image with GPU support
+FROM tensorflow/tensorflow:latest-gpu-jupyter
+
+# Install necessary packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -s /bin/bash oleg
+
+RUN mkdir -p /workspace/tf_cache
+
+# Install packages
+RUN pip install tensorflow-gpu
+RUN pip install jupyterlab
+RUN pip install plotly
+
+# Install additional applications
+RUN apt update
+RUN apt-get install graphviz -y
+
+# Set the working directory
+WORKDIR /workspace
+
+# Expose the port for JupyterLab
+EXPOSE ${JUPYTER_PORT}
+
+# Command to run JupyterLab
+CMD ["jupyter-lab", "--ip=0.0.0.0", "--port=9999", "--no-browser", "--allow-root", "--NotebookApp.token=''"]

tensorflow/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.8'
+
+#==================================================
+# Relies on the file `.env` content for varables:
+#    ROOT_DIR
+#    JUPYTER_PORT
+#==================================================
+
+services:
+  jupyter:
+    user: "oleg"
+    build: ${ROOT_DIR}
+    image: cvtt/jupyter_gpu_tf:v1.0.2
+    container_name: jupyter_gpu_tf
+    runtime: nvidia
+    environment:
+      - JUPYTER_ENABLE_LAB=yes
+      - TF_CACHE_DIR=/workspace/tf_cache
+    volumes:
+      - ${ROOT_DIR}/notebooks:/workspace
+      - ${ROOT_DIR}/jupyter_settings/user-settings:/home/oleg/.jupyter/lab/user-settings
+      - ${ROOT_DIR}/jupyter_settings/workspaces:/home/oleg/.jupyter/lab/workspaces
+    ports:
+      - "${JUPYTER_PORT}:${JUPYTER_PORT}"
+    shm_size: "8gb"

tensorflow/notebooks/Testing GPU.ipynb

@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "000d047d-3dfd-48cb-a3c3-bd1ad7c78b71",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TensorFlow version: 2.16.1\n",
+      "GPUs detected: 1\n",
+      " - /physical_device:GPU:0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-06-03 21:52:43.785816: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-06-03 21:52:43.795907: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-06-03 21:52:43.799970: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tensorflow as tf\n",
+    "\n",
+    "def get_available_gpus():\n",
+    "    gpus = tf.config.list_physical_devices('GPU')\n",
+    "    if gpus:\n",
+    "        print(f\"GPUs detected: {len(gpus)}\")\n",
+    "        for gpu in gpus:\n",
+    "            print(f\" - {gpu.name}\")\n",
+    "    else:\n",
+    "        print(\"No GPUs detected.\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    print(\"TensorFlow version:\", tf.__version__)\n",
+    "    get_available_gpus()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "324dc7a1-61b8-4384-bfed-4385e7dbe49d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0rc1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}