research/jupyter_gpu
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Oleg Sheynin
2024-06-07 00:47:34 +00:00
parent da781060e0
commit 275bdcada6
8 changed files with 1010 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pytorch/.gitignore
+1
View File
@@ -0,0 +1 @@
**/AutogluonModels/
pytorch/docker-compose.yml
+3 -1
View File
@@ -8,17 +8,19 @@ version: '3.8'
services:
jupyter:
build: ${ROOT_DIR}
image: cvtt/jupyter_gpu_pt:v1.1.2
image: cvtt/jupyter_gpu_pt:v1.1.3
container_name: jupyter_gpu_pt
runtime: nvidia
environment:
- JUPYTER_ENABLE_LAB=yes
- PYTHONPATH=/cvtt/prod
volumes:
- ${ROOT_DIR}/notebooks:/workspace
- ${ROOT_DIR}/jupyter_settings/user-settings:/root/.jupyter/lab/user-settings
- ${ROOT_DIR}/jupyter_settings/workspaces:/root/.jupyter/lab/workspaces
- ${ROOT_DIR}/.cache/torch:/root/.cache/torch
- /opt/jupyter_gpu/data:/workspace/data
- ${ROOT_DIR}/cvtt:/cvtt
ports:
- "${JUPYTER_PORT}:${JUPYTER_PORT}"
shm_size: "8gb"
pytorch/get_version.sh
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/bin/bash
function usage() {
echo "Usage: ${0} <project> <version>"
exit 1
}
# ----- Settings
Root=/opt/jupyter_gpu/pytorch/cvtt
LocalSoftwareDir=${Root}/software/cvtt2
ProdDir=${Root}/prod
ReleaseHosts=("cloud21.cvtt.vpn")
ReleasePorts=("22")
ReleaseUsers=("cvttdist")
ReleaseDir=("/home/cvttdist/software/cvtt2")
# ----- Settings
Project=${1}
Version=${2}
if [ "${Project}" == "" ]
then
usage
fi
if [ "${Version}" == "" ]
then
Version=latest
fi
function rsync_load_version() {
for idx in "${!ReleaseHosts[@]}"
do
host=${ReleaseHosts[${idx}]}
port=${ReleasePorts[${idx}]}
user=${ReleaseUsers[${idx}]}
rel_dir=${ReleaseDir[${idx}]}
if [ "${Version}" == "latest" ]; then
echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
echo "Latest version is ${Version}"
fi
echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
then
echo "Directory found..."
rsync_cmd="rsync -ahvv -e \"ssh -p ${port}\""
rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}"
rsync_cmd="${rsync_cmd} ${LocalSoftwareDir}/${Project}/"
echo ${rsync_cmd}
eval ${rsync_cmd}
status=$?
if [ ${status} -eq 0 ]
then
echo "Loading successful..."
break
fi
else
echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
fi
done
if [[ ! -d ${LocalSoftwareDir}/${Project} ]] ; then
echo ERROR loading software
exit 1
fi
}
mkdir -p ${LocalSoftwareDir}
mkdir -p ${ProdDir}
# exists and not empty
rsync_load_version
RelLocalSoftwareDir=../software/cvtt2
Location="${RelLocalSoftwareDir}/${Project}/${Version}/${Project}"
Cmd="cd ${ProdDir}"
Cmd="${Cmd} && rm -rf ${Project}"
Cmd="${Cmd} && ln -snf ${Location} ${Project}"
echo ${Cmd} && eval ${Cmd}
echo "Done: $0 $*"
pytorch/notebooks/Testing GPU.ipynb
+73 -65
View File
@@ -1,8 +1,16 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "53fb9dcb-843b-4633-b031-22236bfeb815",
"metadata": {},
"source": [
"### Check CUDA device"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 8,
"id": "6b269e64-be58-43b5-ad60-0fbd1d37861a",
"metadata": {},
"outputs": [
@@ -27,90 +35,95 @@
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "97906ea2-b284-4966-9c11-b8629f053815",
"cell_type": "markdown",
"id": "e6b4bc89-4cbd-4d4a-99a8-2d5bfb5d095d",
"metadata": {},
"outputs": [],
"source": [
"import plotly"
"### Environment"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 10,
"id": "7313a620-a0eb-4207-a12a-90aeee3cd980",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]'"
"('3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]',\n",
" environ{'PATH': '/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',\n",
" 'HOSTNAME': '6c24ffc52409',\n",
" 'JUPYTER_ENABLE_LAB': 'yes',\n",
" 'PYTHONPATH': '/cvtt/prod',\n",
" 'NVIDIA_VISIBLE_DEVICES': 'all',\n",
" 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility',\n",
" 'LD_LIBRARY_PATH': '/usr/local/nvidia/lib:/usr/local/nvidia/lib64',\n",
" 'PYTORCH_VERSION': '2.2.1',\n",
" 'HOME': '/root',\n",
" 'LC_CTYPE': 'C.UTF-8',\n",
" 'JPY_SESSION_NAME': '/workspace/Testing GPU.ipynb',\n",
" 'JPY_PARENT_PID': '1',\n",
" 'PYDEVD_USE_FRAME_EVAL': 'NO',\n",
" 'TERM': 'xterm-color',\n",
" 'CLICOLOR': '1',\n",
" 'FORCE_COLOR': '1',\n",
" 'CLICOLOR_FORCE': '1',\n",
" 'PAGER': 'cat',\n",
" 'GIT_PAGER': 'cat',\n",
" 'MPLBACKEND': 'module://matplotlib_inline.backend_inline',\n",
" 'CUDA_MODULE_LOADING': 'LAZY'})"
]
},
"execution_count": 3,
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sys\n",
"sys.version"
"sys.version,os.environ"
]
},
{
"cell_type": "markdown",
"id": "943ac637-42c7-4a69-a6c4-94c382e22653",
"metadata": {},
"source": [
"### Test packages avai"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 11,
"id": "95d9a2e6-3464-4dbe-9a97-0c2d5eb34193",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"environ{'PATH': '/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',\n",
" 'HOSTNAME': '1fc69c311e22',\n",
" 'JUPYTER_ENABLE_LAB': 'yes',\n",
" 'NVIDIA_VISIBLE_DEVICES': 'all',\n",
" 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility',\n",
" 'LD_LIBRARY_PATH': '/usr/local/nvidia/lib:/usr/local/nvidia/lib64',\n",
" 'PYTORCH_VERSION': '2.2.1',\n",
" 'HOME': '/root',\n",
" 'LC_CTYPE': 'C.UTF-8',\n",
" 'JPY_SESSION_NAME': '/workspace/oleg/Testing GPU.ipynb',\n",
" 'JPY_PARENT_PID': '1',\n",
" 'PYDEVD_USE_FRAME_EVAL': 'NO',\n",
" 'TERM': 'xterm-color',\n",
" 'CLICOLOR': '1',\n",
" 'FORCE_COLOR': '1',\n",
" 'CLICOLOR_FORCE': '1',\n",
" 'PAGER': 'cat',\n",
" 'GIT_PAGER': 'cat',\n",
" 'MPLBACKEND': 'module://matplotlib_inline.backend_inline',\n",
" 'CUDA_MODULE_LOADING': 'LAZY'}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"import os\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"import matplotlib\n",
"os.environ"
"import plotly\n",
"\n",
"import cvttpy.tools.timeutils as tu\n",
"\n",
"import autogluon\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 12,
"id": "eb38de31-fc19-4515-b08d-9cd7607ea958",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "75642bda0eed47598e340b8c1766949a",
"model_id": "2b2a98cdd61d477b811279a6753630a2",
"version_major": 2,
"version_minor": 0
},
@@ -127,42 +140,37 @@
"text": [
"Done\n"
]
},
{
"data": {
"text/plain": [
"1717721060059418080"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import ipywidgets\n",
"from ipywidgets import interact, IntSlider\n",
"import cvttpy.tools.timeutils as tu\n",
"\n",
"def f(x):\n",
" return x\n",
"\n",
"interact(f, x=IntSlider(min=0, max=10, step=1, value=5))\n",
"print(\"Done\")\n"
"print(\"Done\")\n",
"tu.current_nanoseconds()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"id": "f46e46a7-9b57-44aa-9bc9-dcbcf643bc88",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"jupyter-events==0.10.0\n",
"jupyter-lsp==2.2.5\n",
"jupyter_client==8.6.2\n",
"jupyter_core==5.7.2\n",
"jupyter_server==2.14.1\n",
"jupyter_server_terminals==0.5.3\n",
"jupyterlab==4.2.1\n",
"jupyterlab_pygments==0.3.0\n",
"jupyterlab_server==2.27.2\n",
"jupyterlab_widgets==3.0.11\n"
]
}
],
"outputs": [],
"source": [
"!pip freeze | grep jupyter"
]
pytorch/notebooks/autogluon_chronos.ipynb
+391
View File
File diff suppressed because one or more lines are too long