Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 72d5a849be | |||
| 6125b10935 | |||
| c637faf4bf | |||
| a8f3b5c495 | |||
| 26d0b1a4e0 | |||
| 2c26e4d258 | |||
| 0624f35ddd | |||
| 7052563d3b | |||
| fb60f5f4f6 | |||
| 98fe0e08f3 | |||
| 0b2d9e40ee | |||
| 3eec3e6bf8 | |||
| 0eded0e6da | |||
| 96247c1248 | |||
| f61518ae20 | |||
| 25fd9ab3c4 | |||
| 66b5cd16dd | |||
| 1db0d08547 | |||
| b61c3a684a | |||
| 78382df12d | |||
| bbfb7112bb | |||
| 23603ad076 | |||
| 81fe6372e9 | |||
| 48f77e5280 | |||
| 2c88bc613c | |||
| 085f1af3da | |||
| fa93450b3d | |||
| 4188dd7c7c | |||
| 929953c1c4 | |||
| e562716b73 | |||
| 7c806107ca |
Executable
+15
@@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/env bash
|
||||||
|
|
||||||
|
Registry=http://homestore.cvtt.vpn:5500
|
||||||
|
|
||||||
|
Catalog=${Registry}/v2/_catalog
|
||||||
|
|
||||||
|
jstr=$(curl -s -X GET ${Catalog})
|
||||||
|
|
||||||
|
# echo ${jstr}
|
||||||
|
|
||||||
|
echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
|
||||||
|
TagListURL="${Registry}/v2/${repo}/tags/list"
|
||||||
|
# echo $repo
|
||||||
|
curl -s -X GET ${TagListURL}
|
||||||
|
done
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
[supervisord]
|
||||||
|
environment=PYTHONPATH="/home/cvtt/prod"
|
||||||
|
autostart=true
|
||||||
|
autorestart=unexpected
|
||||||
|
startretries=3
|
||||||
|
logfile=/var/log/supervisor/supervisord_cvtt.log
|
||||||
|
|
||||||
|
[supervisorctl]
|
||||||
|
|
||||||
|
[inet_http_server]
|
||||||
|
port = 127.0.0.1:9001
|
||||||
|
|
||||||
|
[rpcinterface:supervisor]
|
||||||
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
|
[program:coinbase_rec_cvttdata]
|
||||||
|
directory=/home/cvtt/prod
|
||||||
|
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||||
|
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||||
|
--active_exchanges=COINBASE_AT
|
||||||
|
--instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD
|
||||||
|
--db_credentials_key=TSDB_MD_CVTTDATA
|
||||||
|
--admin_port=7223
|
||||||
|
--log_level=INFO
|
||||||
|
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CVTTDATA.log
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
startretries=1
|
||||||
|
startsecs=3
|
||||||
|
user=cvtt
|
||||||
|
#
|
||||||
|
[program:bnbfut_rec_cvttdata]
|
||||||
|
directory=/home/cvtt/prod
|
||||||
|
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||||
|
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||||
|
--active_exchanges=BNBFUT
|
||||||
|
--instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT
|
||||||
|
--db_credentials_key=TSDB_MD_CVTTDATA
|
||||||
|
--admin_port=7224
|
||||||
|
--log_level=INFO
|
||||||
|
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CVTTDATA.log
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
startretries=1
|
||||||
|
startsecs=3
|
||||||
|
user=cvtt
|
||||||
|
|
||||||
|
[program:md_rec_monitor]
|
||||||
|
directory=/home/cvtt/prod
|
||||||
|
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_rec_monitor.py
|
||||||
|
--config=http://cloud16.cvtt.vpn:6789/apps/md_rec_monitor
|
||||||
|
--db=CVTTDATA
|
||||||
|
--admin_port=7225
|
||||||
|
--log_level=INFO
|
||||||
|
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.MONITOR.CVTTDATA.log
|
||||||
|
autostart=true
|
||||||
|
autorestart=true
|
||||||
|
startretries=1
|
||||||
|
startsecs=3
|
||||||
|
user=cvtt
|
||||||
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
Cmd="docker run"
|
|
||||||
Cmd="${Cmd} -d"
|
|
||||||
Cmd="${Cmd} --rm"
|
|
||||||
Cmd="${Cmd} --pull=always"
|
|
||||||
Cmd="${Cmd} --name=cvtt_config_service"
|
|
||||||
Cmd="${Cmd} -p 6789:6789"
|
|
||||||
Cmd="${Cmd} -v /home/cvtt/prod/config_service/data:/app/data"
|
|
||||||
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
|
|
||||||
Cmd="${Cmd} cloud21.cvtt.vpn:5500/config_service:latest"
|
|
||||||
|
|
||||||
echo ${Cmd}
|
|
||||||
eval ${Cmd}
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
date_to_load=${1}
|
date_to_load=${1} # YYYY-MM-DD
|
||||||
|
|
||||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
DockerRegistry=cloud21.cvtt.vpn:5500
|
||||||
DockerImage=${DockerRegistry}/alpaca_md_day #:latest
|
DockerImage=${DockerRegistry}/alpaca_md_day #:latest
|
||||||
@@ -46,7 +46,7 @@ get_prev_business_day() {
|
|||||||
export -f get_prev_business_day
|
export -f get_prev_business_day
|
||||||
|
|
||||||
if [ -z ${date_to_load} ] ; then
|
if [ -z ${date_to_load} ] ; then
|
||||||
echo "Yesterday data has priority. Running historical container will be stopped"
|
echo "Date is not specified. Yesterday data has priority. Running historical container will be stopped"
|
||||||
Cmd="docker stop ${ContainerName}"
|
Cmd="docker stop ${ContainerName}"
|
||||||
echo ${Cmd} && eval ${Cmd}
|
echo ${Cmd} && eval ${Cmd}
|
||||||
Cmd="docker kill ${ContainerName}"
|
Cmd="docker kill ${ContainerName}"
|
||||||
@@ -62,6 +62,7 @@ if [ -z ${date_to_load} ] ; then
|
|||||||
exit 3
|
exit 3
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
|
date_to_load=$(date -d ${date_to_load} '+%Y-%m-%d') #expected format
|
||||||
echo "Historical run ${date_to_load}"
|
echo "Historical run ${date_to_load}"
|
||||||
if is_container_running "$ContainerName"; then
|
if is_container_running "$ContainerName"; then
|
||||||
echo "Container ${ContainerName} is already running."
|
echo "Container ${ContainerName} is already running."
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
ROOT_DIR=/home/cvtt/prod
|
|
||||||
ADMIN_PORT=7220
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
version: '3.8'
|
|
||||||
|
|
||||||
#==================================================
|
|
||||||
# Relies on the file `.env` content for varables:
|
|
||||||
# ROOT_DIR
|
|
||||||
# JUPYTER_PORT
|
|
||||||
#==================================================
|
|
||||||
services:
|
|
||||||
md_portal:
|
|
||||||
image: cloud21.cvtt.vpn:5500/md_portal:latest
|
|
||||||
network_mode: host
|
|
||||||
container_name: md_portal
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
- JUPYTER_ENABLE_LAB=yes
|
|
||||||
volumes:
|
|
||||||
- ${ROOT_DIR}/logs:/logs
|
|
||||||
ports:
|
|
||||||
- "${ADMIN_PORT}:${ADMIN_PORT}"
|
|
||||||
shm_size: "8gb"
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
args=${*}
|
|
||||||
|
|
||||||
ConfigServer=cloud23.cvtt.vpn:6789
|
|
||||||
ActiveExchanges=ALPACA_SNDBX
|
|
||||||
PortalName=MD_PORTAL_ALPACA_SNDBX
|
|
||||||
AdminPort=7220
|
|
||||||
|
|
||||||
|
|
||||||
while getopts ":c:e:a:n:" opt; do
|
|
||||||
case ${opt} in
|
|
||||||
c )
|
|
||||||
ConfigServer=$OPTARG
|
|
||||||
;;
|
|
||||||
e )
|
|
||||||
ActiveExchanges=$OPTARG
|
|
||||||
;;
|
|
||||||
a )
|
|
||||||
AdminPort=$OPTARG
|
|
||||||
;;
|
|
||||||
n )
|
|
||||||
PortalName=$OPTARG
|
|
||||||
;;
|
|
||||||
\? )
|
|
||||||
echo "Invalid option: -$OPTARG" >&2
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
: )
|
|
||||||
echo "Option -$OPTARG requires an argument." >&2
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ConfigServer=${ConfigServer}"
|
|
||||||
echo "ActiveExchanges=${ActiveExchanges}"
|
|
||||||
echo "PortalName=${PortalName}"
|
|
||||||
echo "AdminPort=${AdminPort}"
|
|
||||||
|
|
||||||
|
|
||||||
ImageName=md_portal
|
|
||||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
|
||||||
DockerImage=${DockerRegistry}/${ImageName}
|
|
||||||
|
|
||||||
Cmd="docker run"
|
|
||||||
Cmd="${Cmd} --detach"
|
|
||||||
Cmd="${Cmd} --restart=unless-stopped"
|
|
||||||
Cmd="${Cmd} --pull=always"
|
|
||||||
Cmd="${Cmd} --network=host"
|
|
||||||
Cmd="${Cmd} --name=${ImageName}.${PortalName}"
|
|
||||||
Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
|
|
||||||
Cmd="${Cmd} ${DockerImage}"
|
|
||||||
Cmd="${Cmd} ${args}"
|
|
||||||
|
|
||||||
echo $Cmd
|
|
||||||
eval $Cmd
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
Cmd="docker run"
|
|
||||||
Cmd="${Cmd} -d"
|
|
||||||
#Cmd="${Cmd} --rm"
|
|
||||||
Cmd="${Cmd} --pull=always"
|
|
||||||
Cmd="${Cmd} --name=relative_liquidity_svc"
|
|
||||||
Cmd="${Cmd} -p 5678:5678"
|
|
||||||
Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
|
|
||||||
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
|
|
||||||
Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
|
|
||||||
|
|
||||||
echo ${Cmd}
|
|
||||||
eval ${Cmd}
|
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
echo -n "Usage: $0"
|
|
||||||
echo -n " -b <book>"
|
|
||||||
echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
|
|
||||||
echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
|
|
||||||
echo -n " [-a <admin_port (7223)>"]
|
|
||||||
echo
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
args=${*}
|
|
||||||
|
|
||||||
ConfigServer=cloud23.cvtt.vpn:6789
|
|
||||||
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
|
|
||||||
AdminPort=7223
|
|
||||||
Book=""
|
|
||||||
|
|
||||||
|
|
||||||
while getopts ":b:c:e:a:" opt; do
|
|
||||||
case ${opt} in
|
|
||||||
c )
|
|
||||||
ConfigServer=$OPTARG
|
|
||||||
;;
|
|
||||||
e )
|
|
||||||
ActiveExchanges=$OPTARG
|
|
||||||
;;
|
|
||||||
a )
|
|
||||||
AdminPort=$OPTARG
|
|
||||||
;;
|
|
||||||
b )
|
|
||||||
Book=$OPTARG
|
|
||||||
;;
|
|
||||||
\? )
|
|
||||||
echo "Invalid option: -$OPTARG" >&2
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
: )
|
|
||||||
echo "Option -$OPTARG requires an argument." >&2
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
|
|
||||||
echo "ConfigServer=${ConfigServer}"
|
|
||||||
echo "ActiveExchanges=${ActiveExchanges}"
|
|
||||||
echo "Book=${Book}"
|
|
||||||
echo "AdminPort=${AdminPort}"
|
|
||||||
|
|
||||||
if [ "${Book}" == "" ]; then
|
|
||||||
echo "Book is missing"
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
ImageName=risk_mgr
|
|
||||||
DockerRegistry=cloud21.cvtt.vpn:5500
|
|
||||||
DockerImage=${DockerRegistry}/${ImageName}
|
|
||||||
|
|
||||||
Cmd="docker run"
|
|
||||||
Cmd="${Cmd} --detach"
|
|
||||||
Cmd="${Cmd} --restart=unless-stopped"
|
|
||||||
Cmd="${Cmd} --pull=always"
|
|
||||||
Cmd="${Cmd} --network=host"
|
|
||||||
Cmd="${Cmd} --name=${ImageName}.${Book}"
|
|
||||||
Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
|
|
||||||
Cmd="${Cmd} ${DockerImage}"
|
|
||||||
Cmd="${Cmd} ${args}"
|
|
||||||
|
|
||||||
echo $Cmd
|
|
||||||
eval $Cmd
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
|
|
||||||
#echo ${Cmd} && eval ${Cmd}
|
|
||||||
|
|
||||||
Cmd="docker run"
|
|
||||||
Cmd="${Cmd} -d"
|
|
||||||
Cmd="${Cmd} --rm"
|
|
||||||
Cmd="${Cmd} --pull=always"
|
|
||||||
Cmd="${Cmd} --name trading-calendar"
|
|
||||||
Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
|
|
||||||
echo ${Cmd} && eval ${Cmd}
|
|
||||||
+1
-1
@@ -1 +1 @@
|
|||||||
1.5.4,fix: md_portal docker start
|
1.7.1,alpaca md hist convertitn date to the right format to be used for trading calendar
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ if [ "${host}" == "cloud21" ] ; then
|
|||||||
SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
|
SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
|
||||||
elif [ "${host}" == "homestore" ]; then
|
elif [ "${host}" == "homestore" ]; then
|
||||||
SourceHost=homestore.cvtt.vpn
|
SourceHost=homestore.cvtt.vpn
|
||||||
SourceUser=oleg
|
SourceUser=cvtt
|
||||||
SourceRootDir=/works/cvtt/md_archive/crypto/${source}
|
SourceRootDir=/works/cvtt/md_archive/crypto/${source}
|
||||||
else
|
else
|
||||||
usage
|
usage
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ usage() {
|
|||||||
echo
|
echo
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ "$(uname)" == "Darwin" ]]; then
|
if [[ "$(uname)" == "Darwin" ]]; then
|
||||||
# macOS
|
# macOS
|
||||||
date='gdate'
|
date='gdate'
|
||||||
@@ -22,6 +23,8 @@ host=homestore
|
|||||||
stocks=COIN,GBTC,SQ
|
stocks=COIN,GBTC,SQ
|
||||||
|
|
||||||
TargetDir="/opt/jupyter_gpu/data/eqty_md"
|
TargetDir="/opt/jupyter_gpu/data/eqty_md"
|
||||||
|
# TargetDir="/tmp" # -------= D E B U G
|
||||||
|
|
||||||
mkdir -p ${TargetDir}
|
mkdir -p ${TargetDir}
|
||||||
|
|
||||||
Table=md_1min_bars
|
Table=md_1min_bars
|
||||||
@@ -88,7 +91,6 @@ TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
|
|||||||
|
|
||||||
TargetFilePath="${TargetDir}/${TargetFile}"
|
TargetFilePath="${TargetDir}/${TargetFile}"
|
||||||
mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
|
mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
|
||||||
touch ${TargetFilePath}
|
|
||||||
|
|
||||||
# ---- temp dir
|
# ---- temp dir
|
||||||
echo ${SourceFile}
|
echo ${SourceFile}
|
||||||
@@ -108,7 +110,7 @@ if [ "${host}" == "cloud21" ] ; then
|
|||||||
SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
|
SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
|
||||||
elif [ "${host}" == "homestore" ]; then
|
elif [ "${host}" == "homestore" ]; then
|
||||||
SourceHost=homestore.cvtt.vpn
|
SourceHost=homestore.cvtt.vpn
|
||||||
SourceUser=oleg
|
SourceUser=cvtt
|
||||||
SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
|
SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
|
||||||
else
|
else
|
||||||
usage
|
usage
|
||||||
@@ -146,6 +148,7 @@ function download_file {
|
|||||||
echo ${Cmd} && eval ${Cmd}
|
echo ${Cmd} && eval ${Cmd}
|
||||||
|
|
||||||
DbFile=${tmp_dir}/${SourceFile}
|
DbFile=${tmp_dir}/${SourceFile}
|
||||||
|
touch ${TargetFilePath}
|
||||||
|
|
||||||
|
|
||||||
# --- T E M P - for files older than 20240813
|
# --- T E M P - for files older than 20240813
|
||||||
@@ -173,8 +176,10 @@ for stock in "${Stocks[@]}"; do
|
|||||||
download_file ${SourceFilePath}
|
download_file ${SourceFilePath}
|
||||||
done
|
done
|
||||||
|
|
||||||
chmod 600 ${TargetFilePath}
|
if [ -f ${TargetFilePath} ]; then
|
||||||
ls -lh ${TargetFilePath}
|
chmod 600
|
||||||
|
ls -lh ${TargetFilePath}
|
||||||
|
fi
|
||||||
|
|
||||||
# mkdir -p /tmp/aaa
|
# mkdir -p /tmp/aaa
|
||||||
# cp -r ${tmp_dir}/* /tmp/aaa/
|
# cp -r ${tmp_dir}/* /tmp/aaa/
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
|
Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3.12
|
||||||
RootDir=/home/cvtt/prod
|
RootDir=/home/cvtt/prod
|
||||||
export PYTHONPATH=${RootDir}
|
export PYTHONPATH=${RootDir}
|
||||||
|
|
||||||
|
|||||||
Executable
+88
@@ -0,0 +1,88 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo -n "Usage: ${0}"
|
||||||
|
echo -n " [-c <config (dflt: apps/cvtt_eqt_alpaca)>]"
|
||||||
|
echo -n " [-s <config_serverice (dflt: http://cloud23.cvtt.vpn:6789)>]"
|
||||||
|
echo -n " [-N <name (dflt: ALPACA-SNDBX)>]"
|
||||||
|
echo
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Python="${HOME}/.pyenv/python3.12-venv/bin/python3.12"
|
||||||
|
RootDir="${HOME}/prod"
|
||||||
|
|
||||||
|
# **** D E B U G
|
||||||
|
RootDir=/home/oleg/develop/cvtt2
|
||||||
|
# **** D E B U G
|
||||||
|
|
||||||
|
|
||||||
|
export PYTHONPATH=${RootDir}
|
||||||
|
|
||||||
|
StatusChannel=Status-CVTT
|
||||||
|
AlertChannel=Alerts-CVTT
|
||||||
|
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||||
|
ConfigService=http://cloud23.cvtt.vpn:6789
|
||||||
|
Config=apps/cvtt_eqt_alpaca
|
||||||
|
Name="ALPACA-SNDBX"
|
||||||
|
|
||||||
|
echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
|
||||||
|
|
||||||
|
while getopts ":s:c:n:" opt; do
|
||||||
|
case ${opt} in
|
||||||
|
c )
|
||||||
|
Config=$OPTARG
|
||||||
|
;;
|
||||||
|
s )
|
||||||
|
ConfigService=$OPTARG
|
||||||
|
;;
|
||||||
|
n )
|
||||||
|
Name=$OPTARG
|
||||||
|
;;
|
||||||
|
\? )
|
||||||
|
echo "Invalid option: -$OPTARG" >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
: )
|
||||||
|
echo "Option -$OPTARG requires an argument." >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# ----- For DEBUGGING
|
||||||
|
# Sender=cat
|
||||||
|
# StatusChannel=
|
||||||
|
|
||||||
|
tmpfile=$(mktemp)
|
||||||
|
function cleanup {
|
||||||
|
# echo Cleaing up temporary files: ${TempFiles}
|
||||||
|
if [ "" != "${TempFiles}" ]; then
|
||||||
|
rm -f ${TempFiles}
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
|
||||||
|
Cmd="${Python}"
|
||||||
|
Cmd="${Cmd} ${RootDir}/cvttpy/apps/utils/services_checker.py"
|
||||||
|
Cmd="${Cmd} --config=${ConfigService}/${Config}"
|
||||||
|
Cmd="${Cmd} --log_level=ERROR"
|
||||||
|
Cmd="${Cmd} --log_stdout"
|
||||||
|
Cmd="${Cmd} | tee ${tmpfile} 2>&1"
|
||||||
|
|
||||||
|
echo ${Cmd}
|
||||||
|
eval ${Cmd}
|
||||||
|
|
||||||
|
cat ${tmpfile}
|
||||||
|
exit
|
||||||
|
|
||||||
|
|
||||||
|
# echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
|
||||||
|
# echo >> ${tmpfile}
|
||||||
|
# echo "| host | filesystem | usage |" >> ${tmpfile}
|
||||||
|
# echo "| --- | --- | --- |" >> ${tmpfile}
|
||||||
|
# cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
|
||||||
|
# cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||||
|
|
||||||
@@ -45,7 +45,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
|
|||||||
# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
|
# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
|
||||||
|
|
||||||
export PYTHONPATH=/home/cvtt/prod
|
export PYTHONPATH=/home/cvtt/prod
|
||||||
export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
|
export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3
|
||||||
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
||||||
export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
export PYTHONPATH=${HOME}/prod
|
export PYTHONPATH=${HOME}/prod
|
||||||
|
|
||||||
Python=${HOME}/.pyenv/python3.10-venv/bin/python3
|
Python=${HOME}/.pyenv/python3.12-venv/bin/python3
|
||||||
Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
|
Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
|
||||||
DbFile=${HOME}/prod/data/rel_liquidity.db
|
DbFile=${HOME}/prod/data/rel_liquidity.db
|
||||||
Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
|
Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ run_checklist() {
|
|||||||
|
|
||||||
declare -A Commands
|
declare -A Commands
|
||||||
Commands=(
|
Commands=(
|
||||||
["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
|
["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
|
||||||
["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
|
["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
|
||||||
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
|
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
|
||||||
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
|
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
|
||||||
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
|
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
|
||||||
@@ -47,7 +47,7 @@ function cleanup {
|
|||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
|
echo "## :hearts: CRYPTO MD HEALTH CHECK (cvtt-md.cvtt.vpn)" >> ${tmpfile}
|
||||||
echo '```' >> ${tmpfile}
|
echo '```' >> ${tmpfile}
|
||||||
run_checklist >> ${tmpfile}
|
run_checklist >> ${tmpfile}
|
||||||
echo '```' >> ${tmpfile}
|
echo '```' >> ${tmpfile}
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ run_checklist() {
|
|||||||
yr=$(date -d 'yesterday' '+%Y')
|
yr=$(date -d 'yesterday' '+%Y')
|
||||||
|
|
||||||
declare -A Commands
|
declare -A Commands
|
||||||
Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
|
Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
|
||||||
Commands["homestore"]+="; echo"
|
Commands["homestore"]+="; echo"
|
||||||
Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
|
Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
|
||||||
|
|
||||||
Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
|
Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
|
||||||
Commands["cloud21"]+="; echo"
|
Commands["cloud21"]+="; echo"
|
||||||
@@ -51,26 +51,10 @@ function cleanup {
|
|||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
|
echo "## :fire: EQUITY MD HEALTH CHECK (cryptoval4)" >> ${tmpfile}
|
||||||
echo '```' >> ${tmpfile}
|
echo '```' | tee -a ${tmpfile}
|
||||||
run_checklist >> ${tmpfile}
|
run_checklist | tee -a ${tmpfile}
|
||||||
echo '```' >> ${tmpfile}
|
echo '```' | tee -a ${tmpfile}
|
||||||
|
|
||||||
|
echo "Sending result to ${Sender} ${StatusChannel}"
|
||||||
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
exit
|
|
||||||
# homestore
|
|
||||||
echo homestore
|
|
||||||
ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
|
|
||||||
echo
|
|
||||||
ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
|
|
||||||
|
|
||||||
echo cloud21
|
|
||||||
ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
|
|
||||||
echo
|
|
||||||
ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
|
|
||||||
|
|
||||||
echo "------ gpushnik"
|
|
||||||
ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
export PYTHONPATH=/home/cvtt/prod
|
export PYTHONPATH=/home/cvtt/prod
|
||||||
|
|
||||||
Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
|
Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3"
|
||||||
Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
|
Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
|
||||||
Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
|
Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
|
||||||
Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
|
Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ fi
|
|||||||
rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
|
rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
|
||||||
|
|
||||||
# create db files
|
# create db files
|
||||||
PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
|
PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.12-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
|
||||||
|
|
||||||
if [ "$?" != "0" ] ; then
|
if [ "$?" != "0" ] ; then
|
||||||
exit
|
exit
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
Source=/home/cvtt/prod/archive/md_archive/
|
Source=/home/cvtt/prod/archive/md_archive/
|
||||||
Targets=
|
Targets=
|
||||||
Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
|
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
|
||||||
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
|
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user