load EQT md to GPUshnik fix

build/run_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 <project name>"
+    exit 1
+}
+
+prj=${1}
+if [ "" == "${prj}" ] ; then
+    usage
+fi
+
+Cmd="pushd /home/oleg/develop/cvtt2"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && ./build_release.sh -p ${prj}"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pull --tags"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && popd"
+
+echo ${Cmd}
+eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-1.4.6,removing containers for crypto_md_day
+1.5.2,load EQT md to GPUshnik fix

research/load_eqty_md.sh

@@ -1,7 +1,12 @@
 #!/usr/bin/env bash
 
 usage() {
-  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <stocks comma separated>"
+  echo -n "Usage: $0"
+  echo -n " [-h <host (homestore*/cloud21)>]"
+  echo -n " [-d <YYYYMMDD> (yesterday*)]"
+  echo -n " [-s <stocks comma separated>"
+  echo -n " [-t <target directory>"
+  echo
   exit 1
 }
 if [[ "$(uname)" == "Darwin" ]]; then
@@ -11,6 +16,18 @@ else
     date='date'
 fi
 
+# ------------------ Settings
+md_date=""
+host=homestore
+stocks=COIN,GBTC,SQ
+
+TargetDir="/opt/jupyter_gpu/data/eqty_md"
+mkdir -p ${TargetDir}
+
+Table=md_1min_bars
+# ------------------ Settings
+
+
 
 is_valid() {
   local inst=$1
@@ -26,11 +43,8 @@ is_valid() {
   return 1
 }
 
-md_date=""
-host=homestore
-stocks=COIN,GBTC,SQ
 
-while getopts ":h:d:s:" opt; do
+while getopts ":h:d:s:t:u" opt; do
   case ${opt} in
     d )
       md_date=$OPTARG
@@ -41,6 +55,12 @@ while getopts ":h:d:s:" opt; do
     s )
       stocks=$OPTARG
       ;;
+    t )
+      TargetDir=$OPTARG
+      ;;
+    u )
+      usage
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -64,6 +84,12 @@ if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
     usage
 fi
 
+TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
+
+TargetFilePath="${TargetDir}/${TargetFile}"
+mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
+touch ${TargetFilePath}
+
 # ---- temp dir
 echo ${SourceFile}
 tmp_dir=$(mktemp -d)
@@ -109,29 +135,32 @@ function download_file {
     Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
     echo ${Cmd}
     eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
     ls -lh ${tmp_dir}
     Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
     echo ${Cmd} && eval ${Cmd}
-    
-    ls -lh ${tmp_dir}
-    
-    Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+
+    DbFile=${tmp_dir}/${SourceFile}
+
+
+# --- T E M P - for files older than 20240813
+    sqlite3 ${DbFile} <<EOF
+DROP INDEX md_1min_bars_idx;
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+# --- T E M P
+
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
     echo ${Cmd}
     eval ${Cmd}
 
 }
 
-# TargetDir="/opt/jupyter_gpu/data/eqty_md"
-TargetDir="/tmp/aaa"
-mkdir -p ${TargetDir}
-
-TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
-TargetFilePath="${TargetDir}/${TargetFile}"
-rm -f ${TargetFilePath}
-touch ${TargetFilePath}
-
-Table=md_1min_bars
-
 echo "Loading files"
 for stock in "${Stocks[@]}"; do
     StockLetter="${stock:0:1}"

scripts/__deprecated__/bnbspot_md_recorder_start.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="721${InstrSet}"
-Exchange=BNBSPOT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_BNBSPOT
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done

scripts/__deprecated__/coinbase_md_recorder_start.sh

@@ -1,161 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-InstrSetInstrs[2]=PAIR-USDT-USD,PAIR-MATIC-USD
-InstrSetInstrs[3]=PAIR-DOGE-USD,PAIR-AVAX-USD,
-InstrSetInstrs[4]=PAIR-BCH-USD,PAIR-UNI-USD,PAIR-AAVE-USD,PAIR-LINK-USD,PAIR-SOL-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="720${InstrSet}"
-Exchange=COINBASE_AT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_COINBASE
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/config_server_start.sh

@@ -1,128 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=5
-fi
-
-RootDir=/home/cvtt/prod
-CfgSvcDir=${RootDir}/config_service
-
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Name=config_server
-ServerPy="${RootDir}/cvttpy/apps/utils/config_server.py"
-ServicePort=6789
-ServiceDir=${CfgSvcDir}/data
-LogDir=${RootDir}/logs
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${ServerPy}"
-Cmd="${Cmd} --port=${ServicePort}"
-Cmd="${Cmd} --root=${ServiceDir}"
-Cmd="${Cmd} --log_file=${LogDir}/%T.config_service.log"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-function check_it {
-    if !(timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${ServicePort}/__shutdown__ ..."
-        timeout 5 curl -s localhost:${ServicePort}/__shutdown__
-
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/executor_app_start.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-
-# -------------------- S e t t i n g s
-ConfigServiceHost=cloud11.cvtt.vpn
-ConfigServicePort=6789
-
-RootDir=/home/cvtt/prod
-
-ConfigFile="http://${ConfigServiceHost}:${ConfigServicePort}/apps/executor_app"
-
-AdminPort="7210"
-Exchange=COINBASE_AT
-Name=EXECUTOR_APP
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/executor_app.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-        sleep 3
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+
+
+exit
+# homestore
+echo homestore
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo cloud21
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo "------ gpushnik"
+ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'

utils/git/create_remote_repo.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ---------------------
+# user "git" owned remote repository
+#
+# must run on remote server
+# change GitTopDir, HostName and RemoteName
+# ---------------------
+# R e s u l t  E x a m p l e:
+#     sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
+# ---------------------
+#
+
+# ----- S e t t i n g s
+GitTopDir=/ops/store/git
+HostName=cloud21.cvtt.vpn
+RemoteName=origin
+# ----- S e t t i n g s
+
+usage() {
+    echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
+    exit 1
+}
+
+
+RepoName=${1}
+if [ "" == "${RepoName}" ] ; then
+    usage
+fi
+
+RepoParentDir=${GitTopDir}/cvtt2
+if [ "${2}" != "" ] ; then
+    RepoParentDir=${2}
+fi
+
+RepoDir="${RepoParentDir}/${RepoName}.git"
+
+
+Cmd="sudo bash -c 'mkdir ${RepoDir}"
+Cmd="${Cmd} && cd ${RepoDir}"
+Cmd="${Cmd} && git init --bare "
+Cmd="${Cmd} && chown -R git:git ${RepoDir}'"
+echo ${Cmd}
+eval ${Cmd} || exit 1
+
+echo "====================="
+echo "Repository is created. Use this command to add it:"
+echo "     git remote add ${RemoteName} git@${HostName}:${RepoDir}"
+echo "====================="