homestore /works/cvtt ownership moved to cvtt

build/run_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 <project name>"
+    exit 1
+}
+
+prj=${1}
+if [ "" == "${prj}" ] ; then
+    usage
+fi
+
+Cmd="pushd /home/oleg/develop/cvtt2"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && ./build_release.sh -p ${prj}"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pull --tags"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && popd"
+
+echo ${Cmd}
+eval ${Cmd}

docker/md_portal/start_docker_md_portal.sh

@@ -1,38 +1,54 @@
 #!/bin/bash
 
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-e <active_exchanges (ALPACA_SNBOX)>]"
+    echo -n " [-a <admin_port (7220)>]"
+    echo -n " [-n <portal_name (MD_PORTAL_ALPACA)>]"
+    echo
+    exit 1
+}
+
+
 args=${*}
 
+Config=apps/cvtt_eqt_alpaca
 ConfigServer=cloud23.cvtt.vpn:6789
 ActiveExchanges=ALPACA_SNDBX
-PortalName=MD_PORTAL_ALPACA
+PortalName=MD_PORTAL_ALPACA_SNDBX
 AdminPort=7220
 
-
-while getopts ":c:e:a:n:" opt; do
-  case ${opt} in
-    c )
-      ConfigServer=$OPTARG
-      ;;
-    e )
-      ActiveExchanges=$OPTARG
-      ;;
-    a )
-      AdminPort=$OPTARG
-      ;;
-    n )
-      PortalName=$OPTARG
-      ;;
-    \? )
-      echo "Invalid option: -$OPTARG" >&2
-      usage
-      ;;
-    : )
-      echo "Option -$OPTARG requires an argument." >&2
-      usage
-      ;;
-  esac
+while getopts ":C:c:e:a:n:" opt; do
+    case ${opt} in
+        C )
+            Config=$OPTARG
+            ;;
+        c )
+            ConfigServer=$OPTARG
+            ;;
+        e )
+            ActiveExchanges=$OPTARG
+            ;;
+        a )
+            AdminPort=$OPTARG
+            ;;
+        n )
+            PortalName=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
 done
 
+echo "Config=${Config}"
 echo "ConfigServer=${ConfigServer}"
 echo "ActiveExchanges=${ActiveExchanges}"
 echo "PortalName=${PortalName}"
@@ -43,10 +59,6 @@ ImageName=md_portal
 DockerRegistry=cloud21.cvtt.vpn:5500
 DockerImage=${DockerRegistry}/${ImageName}
 
-if [ -z ${date} ] ; then
-    date=$(date -d "yesterday" +%Y%m%d)
-fi
-
 Cmd="docker run"
 Cmd="${Cmd} --detach"
 Cmd="${Cmd} --restart=unless-stopped"

docker/trading/executor/start_docker_executor.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7222)>"]
+  echo
+  exit 1
+}
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7222
+Book=""
+
+while getopts ":c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "AdminPort=${AdminPort}"
+
+
+ImageName=executor
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/quant/start_docker_quant.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+
+  echo -n " -S <strategy> (dflt: TRDALGO_001)"
+
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7224)>"]
+  echo
+  exit 1
+}
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7224
+Strategy=DAILY_STOCK_001
+Book=""
+
+
+while getopts ":b:c:e:a:S:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    S )
+      Strategy=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Strategy=${Strategy}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=quant
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/risk_mgr/start_docker_risk_mgr.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7223
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=risk_mgr
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/trader/start_docker_trader.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+
+  echo -n " -A <algo> (dflt: TRDALGO_001)"
+
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7226
+Algo=TRDALGO_001
+Book=""
+
+
+while getopts ":b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    A )
+      Algo=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "Algo=${Algo}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trader
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

docker/trading/trading_recorder/start_docker_trading_recorder.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " -b <book>"
+  echo -n " [-C <config (dflt: apps/cvtt_eqt_alpaca)>]"
+  echo -n " [-c <config_server (dflt: cloud23.cvtt.vpn:6789)>]"
+  echo -n " [-e <active_exchanges (ALPACA_SNDBX-MDPORTAL)>]"
+  echo -n " [-a <admin_port (7223)>"]
+  echo
+  exit 1
+}
+
+
+args=${*}
+
+# ConfigServer=cloud16.cvtt.vpn
+ConfigServer=cloud23.cvtt.vpn:6789
+ActiveExchanges=ALPACA_SNDBX-MDPORTAL
+AdminPort=7225
+Book=""
+Config=apps/cvtt_eqt_alpaca
+
+while getopts ":C:b:c:e:a:" opt; do
+  case ${opt} in
+    c )
+      ConfigServer=$OPTARG
+      ;;
+    e )
+      ActiveExchanges=$OPTARG
+      ;;
+    a )
+      AdminPort=$OPTARG
+      ;;
+    b )
+      Book=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+echo "Config=${Config}"
+echo "ConfigServer=${ConfigServer}"
+echo "ActiveExchanges=${ActiveExchanges}"
+echo "Book=${Book}"
+echo "AdminPort=${AdminPort}"
+
+if [ "${Book}" == "" ]; then
+    echo "Book is missing"
+    usage
+fi
+
+
+ImageName=trading_recorder
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/${ImageName}
+
+Cmd="docker run"
+Cmd="${Cmd} --detach"
+Cmd="${Cmd} --restart=unless-stopped"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ImageName}.${Book}"
+Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${args}"
+
+echo $Cmd
+eval $Cmd

release_version.txt

@@ -1 +1 @@
-1.4.7,adding eqty data to research
+1.6.3F2,homestore /works/cvtt ownership moved to cvtt

research/load_crypto_md.sh

@@ -69,7 +69,7 @@ if [ "${host}" == "cloud21" ] ; then
     SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
 elif [ "${host}" == "homestore" ]; then
     SourceHost=homestore.cvtt.vpn
-    SourceUser=oleg
+    SourceUser=cvtt
     SourceRootDir=/works/cvtt/md_archive/crypto/${source}
 else
     usage

research/load_eqty_md.sh

@@ -9,6 +9,7 @@ usage() {
   echo
   exit 1
 }
+
 if [[ "$(uname)" == "Darwin" ]]; then
     # macOS
     date='gdate'
@@ -22,6 +23,8 @@ host=homestore
 stocks=COIN,GBTC,SQ
 
 TargetDir="/opt/jupyter_gpu/data/eqty_md"
+# TargetDir="/tmp"  # -------= D E B U G
+
 mkdir -p ${TargetDir}
 
 Table=md_1min_bars
@@ -88,8 +91,6 @@ TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
 
 TargetFilePath="${TargetDir}/${TargetFile}"
 mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
-touch ${TargetFilePath}
-
 
 # ---- temp dir
 echo ${SourceFile}
@@ -109,7 +110,7 @@ if [ "${host}" == "cloud21" ] ; then
     SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
 elif [ "${host}" == "homestore" ]; then
     SourceHost=homestore.cvtt.vpn
-    SourceUser=oleg
+    SourceUser=cvtt
     SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
 else
     usage
@@ -136,11 +137,19 @@ function download_file {
     Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
     echo ${Cmd}
     eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
     ls -lh ${tmp_dir}
     Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
     echo ${Cmd} && eval ${Cmd}
 
     DbFile=${tmp_dir}/${SourceFile}
+    touch ${TargetFilePath}
+
 
 # --- T E M P - for files older than 20240813
     sqlite3 ${DbFile} <<EOF
@@ -149,7 +158,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, excha
 EOF
 # --- T E M P
 
-    Cmd="sqlite3 ${DbFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
     echo ${Cmd}
     eval ${Cmd}
 
@@ -167,8 +176,10 @@ for stock in "${Stocks[@]}"; do
     download_file ${SourceFilePath}
 done
 
-chmod 600 ${TargetFilePath}
-ls -lh ${TargetFilePath}
+if [ -f ${TargetFilePath} ]; then
+    chmod 600
+    ls -lh ${TargetFilePath}
+fi
 
 # mkdir -p /tmp/aaa
 # cp -r ${tmp_dir}/* /tmp/aaa/

scripts/__deprecated__/bnbspot_md_recorder_start.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="721${InstrSet}"
-Exchange=BNBSPOT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_BNBSPOT
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done

scripts/__deprecated__/coinbase_md_recorder_start.sh

@@ -1,161 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-InstrSetInstrs[2]=PAIR-USDT-USD,PAIR-MATIC-USD
-InstrSetInstrs[3]=PAIR-DOGE-USD,PAIR-AVAX-USD,
-InstrSetInstrs[4]=PAIR-BCH-USD,PAIR-UNI-USD,PAIR-AAVE-USD,PAIR-LINK-USD,PAIR-SOL-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="720${InstrSet}"
-Exchange=COINBASE_AT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_COINBASE
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/config_server_start.sh

@@ -1,128 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=5
-fi
-
-RootDir=/home/cvtt/prod
-CfgSvcDir=${RootDir}/config_service
-
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Name=config_server
-ServerPy="${RootDir}/cvttpy/apps/utils/config_server.py"
-ServicePort=6789
-ServiceDir=${CfgSvcDir}/data
-LogDir=${RootDir}/logs
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${ServerPy}"
-Cmd="${Cmd} --port=${ServicePort}"
-Cmd="${Cmd} --root=${ServiceDir}"
-Cmd="${Cmd} --log_file=${LogDir}/%T.config_service.log"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-function check_it {
-    if !(timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${ServicePort}/__shutdown__ ..."
-        timeout 5 curl -s localhost:${ServicePort}/__shutdown__
-
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/executor_app_start.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-
-# -------------------- S e t t i n g s
-ConfigServiceHost=cloud11.cvtt.vpn
-ConfigServicePort=6789
-
-RootDir=/home/cvtt/prod
-
-ConfigFile="http://${ConfigServiceHost}:${ConfigServicePort}/apps/executor_app"
-
-AdminPort="7210"
-Exchange=COINBASE_AT
-Name=EXECUTOR_APP
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/executor_app.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-        sleep 3
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/healthcheck/cvtt_ts_status_check.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-s <config_serverice (dflt: http://cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-N <name (dflt: ALPACA-SNDBX)>]"
+    echo
+    exit 1
+}
+
+
+Python="${HOME}/.pyenv/python3.10-venv/bin/python3.10"
+RootDir="${HOME}/prod"
+
+# **** D E B U G
+RootDir=/home/oleg/develop/cvtt2
+# **** D E B U G
+
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel=Status-CVTT
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigService=http://cloud23.cvtt.vpn:6789
+Config=apps/cvtt_eqt_alpaca
+Name="ALPACA-SNDBX"
+
+echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
+
+while getopts ":s:c:n:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        s )
+            ConfigService=$OPTARG
+            ;;
+        n )
+            Name=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+tmpfile=$(mktemp)
+function cleanup {
+    # echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
+
+
+Cmd="${Python}"
+Cmd="${Cmd} ${RootDir}/cvttpy/apps/utils/services_checker.py"
+Cmd="${Cmd} --config=${ConfigService}/${Config}"
+Cmd="${Cmd} --log_level=ERROR"
+Cmd="${Cmd} --log_stdout"
+Cmd="${Cmd} | tee ${tmpfile} 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+cat ${tmpfile}
+exit
+
+
+# echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
+# echo  >> ${tmpfile}
+# echo "| host | filesystem | usage |"  >> ${tmpfile}
+# echo "| --- | --- | --- |"  >> ${tmpfile}
+# cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
+# cat ${tmpfile} |  ${Sender} ${StatusChannel}
+

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+exit
+# homestore
+echo homestore
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo cloud21
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo "------ gpushnik"
+ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'

scripts/mkt_data/sync_market_data.sh

@@ -2,7 +2,7 @@
 
 Source=/home/cvtt/prod/archive/md_archive/
 Targets=
-Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
 
 

utils/git/create_remote_repo.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ---------------------
+# user "git" owned remote repository
+#
+# must run on remote server
+# change GitTopDir, HostName and RemoteName
+# ---------------------
+# R e s u l t  E x a m p l e:
+#     sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
+# ---------------------
+#
+
+# ----- S e t t i n g s
+GitTopDir=/ops/store/git
+HostName=cloud21.cvtt.vpn
+RemoteName=origin
+# ----- S e t t i n g s
+
+usage() {
+    echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
+    exit 1
+}
+
+
+RepoName=${1}
+if [ "" == "${RepoName}" ] ; then
+    usage
+fi
+
+RepoParentDir=${GitTopDir}/cvtt2
+if [ "${2}" != "" ] ; then
+    RepoParentDir=${2}
+fi
+
+RepoDir="${RepoParentDir}/${RepoName}.git"
+
+
+Cmd="sudo bash -c 'mkdir ${RepoDir}"
+Cmd="${Cmd} && cd ${RepoDir}"
+Cmd="${Cmd} && git init --bare "
+Cmd="${Cmd} && chown -R git:git ${RepoDir}'"
+echo ${Cmd}
+eval ${Cmd} || exit 1
+
+echo "====================="
+echo "Repository is created. Use this command to add it:"
+echo "     git remote add ${RemoteName} git@${HostName}:${RepoDir}"
+echo "====================="