Compare commits

...

10 Commits

Author SHA1 Message Date
cd5c78d47f . 2024-08-14 01:54:48 +00:00
80319e0301 progress 2024-08-07 21:01:30 +00:00
14519259d8 progress 2024-08-02 21:22:46 +00:00
6b84da949f progress 2024-07-02 15:34:18 +00:00
3dc9d91e48 progress 2024-06-15 00:04:59 +00:00
db4f5679ac . 2024-06-13 20:17:22 +00:00
a74ad29f8f . 2024-06-12 19:53:18 +00:00
a3445075cc progress 2024-06-12 19:10:26 +00:00
bd9c00cc67 . 2024-06-07 01:36:27 +00:00
d6a837e000 progress 2024-06-07 01:31:21 +00:00
19 changed files with 15880 additions and 363 deletions

View File

@ -1,32 +0,0 @@
# -------------------------------------------------------
# Relies on JUPYTER_PORT being set in docker-compose.yml
# -------------------------------------------------------
FROM pytorch/pytorch:latest
# RUN useradd -m -s /bin/bash oleg
# Install packages
# --- this version of jupyter is required to work with the widgets
RUN pip install jupyterlab==3.6.7
#
RUN pip install ipywidgets
RUN pip install pandas numpy plotly
#
RUN pip install fastbook
# Install additional applications
RUN apt update
RUN apt-get install graphviz -y
# Set environment variables, etc.
#ENV MY_ENV_VAR=myvalue
# Set the working directory
WORKDIR /workspace
# Expose the port for JupyterLab
EXPOSE ${JUPYTER_PORT}
# Command to run JupyterLab
CMD ["jupyter-lab", "--ip=0.0.0.0", "--port=7777", "--no-browser", "--allow-root", "--NotebookApp.token=''"]

View File

@ -7,10 +7,12 @@ version: '3.8'
#==================================================
services:
jupyter:
build: ${ROOT_DIR}
image: cvtt/fastai:v0.0.1
# build: ${ROOT_DIR}
# image: cvtt/fastai:v0.0.1
image: cloud21.cvtt.vpn:5500/fastai_jupyter:latest
container_name: fastai_jupyter
runtime: nvidia
restart: unless-stopped
environment:
- JUPYTER_ENABLE_LAB=yes
volumes:
@ -18,7 +20,7 @@ services:
- ${ROOT_DIR}/jupyter_settings/user-settings:/root/.jupyter/lab/user-settings
- ${ROOT_DIR}/jupyter_settings/workspaces:/root/.jupyter/lab/workspaces
- ${ROOT_DIR}/.cache/torch:/root/.cache/torch
# - /opt/jupyter_gpu/data:/workspace/data
ports:
- "${JUPYTER_PORT}:${JUPYTER_PORT}"
shm_size: "8gb"

View File

@ -1,2 +1,3 @@
fastbook
bears
*.jpg

@ -1 +0,0 @@
Subproject commit 054cd8a62c7b6c11f0c36265149ccbaf95d91034

View File

@ -1,57 +0,0 @@
#!/usr/bin/env bash
date=${1}
if [ "${date}" == "" ] ; then
date="yesterday"
fi
SourceHost=cryptoval3.cvtt.vpn
SourceUser=cvtt
SourceRootDir=/localdisk/cvtt/archive/md_archive
SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
SourceFileZip="${SourceFile}.gz"
SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
TargetDir="/opt/jupyter_gpu/data/crypto_md"
TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
TargetFilePath="${TargetDir}/${TargetFile}"
echo ${SourceFile}
tmp_dir=$(mktemp -d)
function cleanup {
cd ${HOME}
rm -rf ${tmp_dir}
}
trap cleanup EXIT
function download_file {
Cmd="rsync"
Cmd="${Cmd} -ahv"
if tty -s; then
Cmd="${Cmd} --progress=info2"
fi
Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
echo ${Cmd}
eval ${Cmd}
ls -lh ${tmp_dir}
Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
echo ${Cmd} && eval ${Cmd}
ls -lh ${tmp_dir}
rm -f ${TargetFilePath}
touch ${TargetFilePath}
for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
do
Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
echo ${Cmd}
eval ${Cmd}
done
chmod 600 ${TargetFilePath}
ls -lh ${TargetFilePath}
}
download_file

View File

@ -1,2 +1,5 @@
ROOT_DIR=/opt/jupyter_gpu/pytorch
DATA_DIR=/opt/jupyter_gpu/data
JUPYTER_PORT=8888
# PUID=1000
# PGID=1000

View File

@ -18,6 +18,7 @@ RUN apt-get install graphviz -y
# Set environment variables, etc.
#ENV MY_ENV_VAR=myvalue
COPY cvttpy /cvttpy
# Set the working directory
WORKDIR /workspace

View File

@ -3,14 +3,18 @@ version: '3.8'
#==================================================
# Relies on the file `.env` content for varables:
# ROOT_DIR
# DATA_DIR
# JUPYTER_PORT
# UID
# GID
#==================================================
services:
jupyter:
build: ${ROOT_DIR}
image: cvtt/jupyter_gpu_pt:v1.1.3
# user: "${UID}:${GID}"
image: cloud21.cvtt.vpn:5500/jupyter_gpu_pytorch:latest
container_name: jupyter_gpu_pt
runtime: nvidia
restart: unless-stopped
environment:
- JUPYTER_ENABLE_LAB=yes
- PYTHONPATH=/cvtt/prod
@ -19,8 +23,8 @@ services:
- ${ROOT_DIR}/jupyter_settings/user-settings:/root/.jupyter/lab/user-settings
- ${ROOT_DIR}/jupyter_settings/workspaces:/root/.jupyter/lab/workspaces
- ${ROOT_DIR}/.cache/torch:/root/.cache/torch
- /opt/jupyter_gpu/data:/workspace/data
- ${ROOT_DIR}/cvtt:/cvtt
- ${DATA_DIR}:/workspace/data
ports:
- "${JUPYTER_PORT}:${JUPYTER_PORT}"
shm_size: "8gb"

View File

@ -0,0 +1,263 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "c7dac550-c0ed-4ec7-846e-8edb2086c9cc",
"metadata": {},
"source": [
"# Augmented Dickey-Fuller Test (ADF)\n",
"Stationarity Test"
]
},
{
"cell_type": "markdown",
"id": "778b9362-37e3-40e0-a20a-1ca5e2cddf05",
"metadata": {},
"source": [
"## Preparing The data"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "998ecc54-aaba-4761-bb98-1eda5c9fa091",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>id</th>\n",
" <th>tstamp</th>\n",
" <th>target</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722470400000000000</td>\n",
" <td>64640.679892</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722470460000000000</td>\n",
" <td>64652.991289</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722470520000000000</td>\n",
" <td>64660.005093</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722470580000000000</td>\n",
" <td>64653.482847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722470640000000000</td>\n",
" <td>64687.458279</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1372</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722556500000000000</td>\n",
" <td>65439.307663</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1373</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722556560000000000</td>\n",
" <td>65445.733114</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1374</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722556620000000000</td>\n",
" <td>65446.371741</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1375</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722556680000000000</td>\n",
" <td>65420.879478</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1376</th>\n",
" <td>PAIR-BTC-USDT</td>\n",
" <td>1722556740000000000</td>\n",
" <td>65377.032222</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1377 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" id tstamp target\n",
"0 PAIR-BTC-USDT 1722470400000000000 64640.679892\n",
"1 PAIR-BTC-USDT 1722470460000000000 64652.991289\n",
"2 PAIR-BTC-USDT 1722470520000000000 64660.005093\n",
"3 PAIR-BTC-USDT 1722470580000000000 64653.482847\n",
"4 PAIR-BTC-USDT 1722470640000000000 64687.458279\n",
"... ... ... ...\n",
"1372 PAIR-BTC-USDT 1722556500000000000 65439.307663\n",
"1373 PAIR-BTC-USDT 1722556560000000000 65445.733114\n",
"1374 PAIR-BTC-USDT 1722556620000000000 65446.371741\n",
"1375 PAIR-BTC-USDT 1722556680000000000 65420.879478\n",
"1376 PAIR-BTC-USDT 1722556740000000000 65377.032222\n",
"\n",
"[1377 rows x 3 columns]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"from statsmodels.tsa.stattools import adfuller\n",
"import numpy as np\n",
"\n",
"def demo_example_data() -> pd.Series:\n",
" # Generate example time series data\n",
" # np.random.seed(0)\n",
" time_series_data = np.random.randn(100) # Random data for demonstration\n",
" \n",
" # Create a pandas Series\n",
" data = pd.Series(time_series_data)\n",
" \n",
" # Optionally, you can add a datetime index if you have time-indexed data\n",
" dates = pd.date_range(start='2020-01-01', periods=len(time_series_data), freq='D')\n",
" data = pd.Series(time_series_data, index=dates)\n",
"\n",
" # Display the first few rows of the data\n",
" print(data.head())\n",
" return data\n",
"\n",
"def load_df_from_db(file: str, query: str) -> pd.DataFrame:\n",
" import sqlite3 \n",
" \n",
" conn = sqlite3.connect(file)\n",
" df = pd.read_sql_query(query, conn)\n",
" df['timestamp'] = pd.to_datetime(df['tstamp'])\n",
" df.set_index('timestamp', inplace=True)\n",
" return df\n",
"\n",
"file_path = \"/workspace/data/crypto_md/20240801.mktdata.ohlcv.db\"\n",
"instrument_id='PAIR-BTC-USDT'\n",
"query = f\"\"\"\n",
"select \n",
" instrument_id as id, \n",
" tstamp, \n",
" vwap \n",
"from bnbspot_ohlcv_1min \n",
"where instrument_id = '{instrument_id}'\n",
"\"\"\"\n",
"\n",
"df = load_df_from_db(file=file_path, query=query)\n",
"df.rename(columns={'vwap': 'target'}, inplace=True)\n",
"# df[\"tstamp2\"] = df.index\n",
"df = df.reset_index()\n",
"df = df.drop([\"timestamp\"], axis=1) \n",
"df"
]
},
{
"cell_type": "markdown",
"id": "43e43154-5a04-4a1d-977c-7f930d62f241",
"metadata": {},
"source": [
"## Running Test"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "b97d357f-787b-4cff-849f-b91b1ec35e7c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-01-01 -1.652908\n",
"2020-01-02 -0.157302\n",
"2020-01-03 -1.396187\n",
"2020-01-04 0.150374\n",
"2020-01-05 1.048603\n",
"Freq: D, dtype: float64\n",
"ADF Statistic: -9.985535987881171\n",
"p-value: 2.060269774403535e-17\n",
"Critical Values: {'1%': -3.498198082189098, '5%': -2.891208211860468, '10%': -2.5825959973472097}\n"
]
}
],
"source": [
"import pandas as pd\n",
"from statsmodels.tsa.stattools import adfuller\n",
"\n",
"# Example time series data\n",
"data = demo_example_data()\n",
"\n",
"# Perform the ADF test\n",
"result = adfuller(data)\n",
"\n",
"# Extract and print the results\n",
"print('ADF Statistic:', result[0])\n",
"print('p-value:', result[1])\n",
"print('Critical Values:', result[4])\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

View File

@ -10,7 +10,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 1,
"id": "6b269e64-be58-43b5-ad60-0fbd1d37861a",
"metadata": {},
"outputs": [
@ -19,7 +19,8 @@
"output_type": "stream",
"text": [
"CUDA is available. Number of GPUs: 1\n",
"GPU Name: NVIDIA GeForce RTX 3060\n"
"GPU Name: NVIDIA GeForce RTX 3060\n",
"GPU Capability: 8\n"
]
}
],
@ -30,6 +31,7 @@
"if torch.cuda.is_available():\n",
" print(f\"CUDA is available. Number of GPUs: {torch.cuda.device_count()}\")\n",
" print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n",
" print(f\"GPU Capability: {torch.cuda.get_device_capability()[0]}\")\n",
"else:\n",
" print(\"CUDA is not available. No GPU detected.\")\n"
]
@ -44,7 +46,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 2,
"id": "7313a620-a0eb-4207-a12a-90aeee3cd980",
"metadata": {},
"outputs": [
@ -53,7 +55,7 @@
"text/plain": [
"('3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]',\n",
" environ{'PATH': '/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',\n",
" 'HOSTNAME': '6c24ffc52409',\n",
" 'HOSTNAME': '30c1a1a6daca',\n",
" 'JUPYTER_ENABLE_LAB': 'yes',\n",
" 'PYTHONPATH': '/cvtt/prod',\n",
" 'NVIDIA_VISIBLE_DEVICES': 'all',\n",
@ -75,14 +77,18 @@
" 'CUDA_MODULE_LOADING': 'LAZY'})"
]
},
"execution_count": 10,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sys\n",
"sys.version,os.environ"
"import os\n",
"\n",
"sys.path.append(\"/\")\n",
"sys.version,os.environ\n",
"\n"
]
},
{
@ -90,12 +96,12 @@
"id": "943ac637-42c7-4a69-a6c4-94c382e22653",
"metadata": {},
"source": [
"### Test packages avai"
"### Test packages availability"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 3,
"id": "95d9a2e6-3464-4dbe-9a97-0c2d5eb34193",
"metadata": {},
"outputs": [],
@ -114,21 +120,29 @@
"\n"
]
},
{
"cell_type": "markdown",
"id": "566016ad-3f13-4a61-b460-39738ae3bf4c",
"metadata": {},
"source": [
"### Jupyter widgets"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 4,
"id": "eb38de31-fc19-4515-b08d-9cd7607ea958",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2b2a98cdd61d477b811279a6753630a2",
"model_id": "54e9ddbe3f4349ca9ebbee5aaec14477",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"interactive(children=(IntSlider(value=5, description='x', max=10), Output()), _dom_classes=('widget-interact',…"
"interactive(children=(IntSlider(value=5, description='x', max=1000), Output()), _dom_classes=('widget-interact…"
]
},
"metadata": {},
@ -144,10 +158,10 @@
{
"data": {
"text/plain": [
"1717721060059418080"
"1723585308075979805"
]
},
"execution_count": 12,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@ -160,17 +174,34 @@
"def f(x):\n",
" return x\n",
"\n",
"interact(f, x=IntSlider(min=0, max=10, step=1, value=5))\n",
"interact(f, x=IntSlider(min=0, max=1000, step=1, value=5))\n",
"print(\"Done\")\n",
"tu.current_nanoseconds()"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"id": "f46e46a7-9b57-44aa-9bc9-dcbcf643bc88",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"jupyter-events==0.10.0\n",
"jupyter-lsp==2.2.5\n",
"jupyter_client==8.6.2\n",
"jupyter_core==5.7.2\n",
"jupyter_server==2.14.2\n",
"jupyter_server_terminals==0.5.3\n",
"jupyterlab==4.2.4\n",
"jupyterlab_pygments==0.3.0\n",
"jupyterlab_server==2.27.3\n",
"jupyterlab_widgets==3.0.11\n"
]
}
],
"source": [
"!pip freeze | grep jupyter"
]

View File

@ -1,2 +1,3 @@
ROOT_DIR=/opt/jupyter_gpu/tensorflow
DATA_DIR=/opt/jupyter_gpu/data
JUPYTER_PORT=9999

View File

@ -17,6 +17,15 @@ RUN pip install jupyterlab
RUN pip install pandas
RUN pip install numpy
RUN pip install plotly
RUN pip install seaborn
RUN pip install statsmodels
RUN pip install scikit-learn
RUN pip install Arch
RUN pip install Quantlib
RUN pip install Backtrader
RUN pip install pyfolio
RUN pip install XGBoost
# Install additional applications
RUN apt update

View File

@ -3,24 +3,25 @@ version: '3.8'
#==================================================
# Relies on the file `.env` content for varables:
# ROOT_DIR
# DATA_DIR
# JUPYTER_PORT
#==================================================
services:
jupyter:
# user: "oleg"
build: ${ROOT_DIR}
image: cvtt/jupyter_gpu_tf:v1.0.5
image: cloud21.cvtt.vpn:5500/jupyter_gpu_tensorflow:latest
container_name: jupyter_gpu_tf
runtime: nvidia
restart: unless-stopped
environment:
- JUPYTER_ENABLE_LAB=yes
#- PYTHONPATH=/cvtt/prod
- TF_CACHE_DIR=/workspace/tf_cache
volumes:
- ${ROOT_DIR}/notebooks:/workspace
- ${ROOT_DIR}/jupyter_settings/user-settings:/home/oleg/.jupyter/lab/user-settings
- ${ROOT_DIR}/jupyter_settings/workspaces:/home/oleg/.jupyter/lab/workspaces
- /opt/jupyter_gpu/data:/workspace/data
- ${DATA_DIR}:/workspace/data
ports:
- "${JUPYTER_PORT}:${JUPYTER_PORT}"
shm_size: "8gb"

View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "89251b34-9738-4fa3-b2db-7d6a7dde7d48",
"metadata": {},
"outputs": [],
@ -24,7 +24,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"id": "eb24b805-ee73-4db4-b1de-0c43bea7d86f",
"metadata": {},
"outputs": [],
@ -42,7 +42,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 5,
"id": "8dcb0ec1-4e55-4d13-91fc-57c24fac6460",
"metadata": {},
"outputs": [
@ -246,7 +246,7 @@
"[5760 rows x 9 columns]"
]
},
"execution_count": 14,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File