update for storage check

admin/docker_images.sh

@@ -0,0 +1,15 @@
+#!/bin/env bash
+
+Registry=http://homestore.cvtt.vpn:5500
+
+Catalog=${Registry}/v2/_catalog
+
+jstr=$(curl -s -X GET ${Catalog})
+
+# echo ${jstr}
+
+echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
+    TagListURL="${Registry}/v2/${repo}/tags/list"
+    # echo $repo
+    curl -s -X GET ${TagListURL}
+done

build/build_release.sh

@@ -180,4 +180,7 @@ do
 done
 
 echo "PROJECT ${project} **** P U L L   N E W   T A G S  F R O M  o r i g i n ****"
+echo "----"
+echo "(cd ${project}; git pull --tags; git pushall)"
+echo "----"
 echo "$0 Done ${project} ${release_version}"

build/run_build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 <project name>"
+    exit 1
+}
+
+prj=${1}
+if [ "" == "${prj}" ] ; then
+    usage
+fi
+
+Cmd="pushd /home/oleg/develop/cvtt2"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && ./build_release.sh -p ${prj}"
+Cmd="${Cmd} && (cd ${prj}"
+Cmd="${Cmd} && git pull --tags"
+Cmd="${Cmd} && git pushall)"
+Cmd="${Cmd} && popd"
+
+echo ${Cmd}
+eval ${Cmd}

doc/supervisord.conf.example

@@ -0,0 +1,61 @@
+[supervisord]
+environment=PYTHONPATH="/home/cvtt/prod"
+autostart=true
+autorestart=unexpected
+startretries=3
+logfile=/var/log/supervisor/supervisord_cvtt.log
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:coinbase_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=COINBASE_AT
+        --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7223
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+#
+[program:bnbfut_rec_cvttdata]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
+        --active_exchanges=BNBFUT
+        --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT
+        --db_credentials_key=TSDB_MD_CVTTDATA
+        --admin_port=7224
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+
+[program:md_rec_monitor]
+directory=/home/cvtt/prod
+command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_rec_monitor.py
+        --config=http://cloud16.cvtt.vpn:6789/apps/md_rec_monitor
+        --db=CVTTDATA
+        --admin_port=7225
+        --log_level=INFO
+        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.MONITOR.CVTTDATA.log
+autostart=true
+autorestart=true
+startretries=1
+startsecs=3
+user=cvtt
+

docker/config_service.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-Cmd="docker run"
-Cmd="${Cmd} -d"
-Cmd="${Cmd} --rm"
-Cmd="${Cmd} --pull=always"
-Cmd="${Cmd} --name=cvtt_config_service"
-Cmd="${Cmd} -p 6789:6789"
-Cmd="${Cmd} -v /home/cvtt/prod/config_service/data:/app/data"
-Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
-Cmd="${Cmd} cloud21.cvtt.vpn:5500/config_service:latest"
-
-echo ${Cmd}
-eval ${Cmd}

docker/market_data/hist/alpaca_md_day.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-date_to_load=${1}
+date_to_load=${1} # YYYY-MM-DD
 
 DockerRegistry=cloud21.cvtt.vpn:5500
 DockerImage=${DockerRegistry}/alpaca_md_day #:latest
@@ -46,7 +46,7 @@ get_prev_business_day() {
 export -f get_prev_business_day
 
 if [ -z ${date_to_load} ] ; then
-    echo "Yesterday data has priority. Running historical container will be stopped"
+    echo "Date is not specified. Yesterday data has priority. Running historical container will be stopped"
     Cmd="docker stop ${ContainerName}"
     echo ${Cmd} && eval ${Cmd}
     Cmd="docker kill ${ContainerName}"
@@ -62,6 +62,7 @@ if [ -z ${date_to_load} ] ; then
         exit 3
     fi
 else
+    date_to_load=$(date -d ${date_to_load} '+%Y-%m-%d') #expected format
     echo "Historical run ${date_to_load}"
     if is_container_running "$ContainerName"; then
         echo "Container ${ContainerName} is already running."

docker/market_data/hist/crypto_md_day.sh

@@ -18,7 +18,8 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"
-Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+# Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+Cmd="${Cmd} -s coinbase,bnbspot"
 
 echo $Cmd
 eval $Cmd

docker/market_data/hist/load_alpaca_hbar.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0  -N <num_symbols> [-L <LogDir>] [-d <YYYYMMDD Date>]"
+  exit 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+# ----- Settings
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_hbar #:latest
+ContainerName=alpaca_hbar
+LogDir=/home/cvtt/prod/logs
+# ----- Settings
+
+while getopts ":d:N:L:" opt; do
+  case ${opt} in
+    d )
+      date_to_load=$OPTARG
+      ;;
+    N )
+      NumSymbols=$OPTARG
+      ;;
+    L )
+      LogDir=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running."
+    exit 3
+fi
+
+if [ -z "${date_to_load}" ]; then
+    echo "date_to_load is empty"
+    date_to_load=$(get_prev_business_day $(date  -d "yesterday" '+%Y-%m-%d'))
+    echo "Historical Data for ${date_to_load}"
+fi
+
+echo "date_to_load=${date_to_load}"
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -d ${date_to_load}"
+if [ -n "${NumSymbols}" ]; then
+    Cmd="${Cmd} -N ${NumSymbols}"
+fi
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi

docker/market_data/hist/load_alpaca_qat.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0  -S <symbols> [-L <LogDir>] [-d <YYYYMMDD Date>]"
+  exit 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+# ----- Settings
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_qat #:latest
+ContainerName=alpaca_qat
+LogDir=/home/cvtt/prod/logs
+# ----- Settings
+
+while getopts ":d:S:L:" opt; do
+  case ${opt} in
+    d )
+      date_to_load=$OPTARG
+      ;;
+    S )
+      Symbols=$OPTARG
+      ;;
+    L )
+      LogDir=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ -z ${Symbols}  ] ; then
+    echo "symbols are not specified"
+    usage
+fi
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running."
+    exit 3
+fi
+
+if [ -z "${date_to_load}" ]; then
+    echo "date_to_load is empty"
+    date_to_load=$(get_prev_business_day $(date  -d "yesterday" '+%Y-%m-%d'))
+    echo "Historical Data for ${date_to_load}"
+fi
+
+echo "date_to_load=${date_to_load}"
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -d ${date_to_load}"
+Cmd="${Cmd} -S ${Symbols}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi

docker/relative_liquidity.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-Cmd="docker run"
-Cmd="${Cmd} -d"
-#Cmd="${Cmd} --rm"
-Cmd="${Cmd} --pull=always"
-Cmd="${Cmd} --name=relative_liquidity_svc"
-Cmd="${Cmd} -p 5678:5678"
-Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
-Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
-Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
-
-echo ${Cmd}
-eval ${Cmd}

docker/trading_calendar.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
-#echo ${Cmd} && eval ${Cmd}
-
-Cmd="docker run"
-Cmd="${Cmd} -d"
-Cmd="${Cmd} --rm"
-Cmd="${Cmd} --pull=always"
-Cmd="${Cmd} --name trading-calendar"
-Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
-echo ${Cmd} && eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-1.3.8,build release script updated
+1.7.4,minor update for storage check

research/load_crypto_md.sh

@@ -69,7 +69,7 @@ if [ "${host}" == "cloud21" ] ; then
     SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
 elif [ "${host}" == "homestore" ]; then
     SourceHost=homestore.cvtt.vpn
-    SourceUser=oleg
+    SourceUser=cvtt
     SourceRootDir=/works/cvtt/md_archive/crypto/${source}
 else
     usage
@@ -77,6 +77,7 @@ fi
 
 echo "${SourceHost} ${SourceUser} ${SourceRootDir}"
 
+
 SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
 SourceFileZip="${SourceFile}.gz"
 SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
@@ -85,6 +86,10 @@ TargetDir="/opt/jupyter_gpu/data/crypto_md"
 TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
 TargetFilePath="${TargetDir}/${TargetFile}"
 
+# Tables=(bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min)
+Tables=(bnbspot_ohlcv_1min coinbase_ohlcv_1min)
+
+
 echo ${SourceFile}
 tmp_dir=$(mktemp -d)
 
@@ -112,7 +117,7 @@ function download_file {
     rm -f ${TargetFilePath}
     touch ${TargetFilePath}
 
-    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    for table in "${Tables[@]}"
     do
         Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
         echo ${Cmd}
@@ -121,6 +126,5 @@ function download_file {
     chmod 600 ${TargetFilePath}
     ls -lh ${TargetFilePath}
 
-
 }
 download_file

research/load_eqty_md.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo -n "Usage: $0"
+  echo -n " [-h <host (homestore*/cloud21)>]"
+  echo -n " [-d <YYYYMMDD> (yesterday*)]"
+  echo -n " [-s <stocks comma separated>"
+  echo -n " [-t <target directory>"
+  echo
+  exit 1
+}
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    # macOS
+    date='gdate'
+else
+    date='date'
+fi
+
+# ------------------ Settings
+md_date=""
+host=homestore
+stocks=COIN,GBTC,SQ
+
+TargetDir="/opt/jupyter_gpu/data/eqty_md"
+# TargetDir="/tmp"  # -------= D E B U G
+
+mkdir -p ${TargetDir}
+
+Table=md_1min_bars
+# ------------------ Settings
+
+
+
+is_valid() {
+  local inst=$1
+  shift
+  local valid_instances=("$@")
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+
+while getopts ":h:d:s:t:u" opt; do
+  case ${opt} in
+    d )
+      md_date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      stocks=$OPTARG
+      ;;
+    t )
+      TargetDir=$OPTARG
+      ;;
+    u )
+      usage
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${md_date}" == "" ] ; then
+    md_date="yesterday"
+fi
+
+echo "$md_date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo "Host '${host}' is not valid"
+    usage
+fi
+
+TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
+
+TargetFilePath="${TargetDir}/${TargetFile}"
+mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
+
+# ---- temp dir
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+# ---- temp dir
+
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
+else
+    usage
+fi
+
+
+OLDIFS=$IFS
+IFS=','
+read -ra Stocks <<< "$stocks"
+IFS=$OLDIFS
+
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir} ${Stocks[@]}"
+
+
+function download_file {
+    local SourceFilePath=${1}
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+
+    DbFile=${tmp_dir}/${SourceFile}
+    touch ${TargetFilePath}
+
+
+# --- T E M P - for files older than 20240813
+    sqlite3 ${DbFile} <<EOF
+DROP INDEX md_1min_bars_idx;
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+# --- T E M P
+
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+}
+
+echo "Loading files"
+for stock in "${Stocks[@]}"; do
+    StockLetter="${stock:0:1}"
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
+
+    SourceFileZip="${SourceFile}.gz"
+    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
+    echo ${SourceFilePath}
+
+    download_file ${SourceFilePath}
+done
+
+if [ -f ${TargetFilePath} ]; then
+    chmod 600
+    ls -lh ${TargetFilePath}
+fi
+
+# mkdir -p /tmp/aaa
+# cp -r ${tmp_dir}/* /tmp/aaa/

scripts/__DEPRECATED__/archive_yesterday_md.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
+Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3.12
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 

scripts/__deprecated__/bnbspot_md_recorder_start.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="721${InstrSet}"
-Exchange=BNBSPOT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_BNBSPOT
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done

scripts/__deprecated__/coinbase_md_recorder_start.sh

@@ -1,161 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-declare -A InstrSetInstrs
-InstrSetInstrs[1]=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-InstrSetInstrs[2]=PAIR-USDT-USD,PAIR-MATIC-USD
-InstrSetInstrs[3]=PAIR-DOGE-USD,PAIR-AVAX-USD,
-InstrSetInstrs[4]=PAIR-BCH-USD,PAIR-UNI-USD,PAIR-AAVE-USD,PAIR-LINK-USD,PAIR-SOL-USD
-
-
-if [ "${InstrSet}" == "" ]
-then
-    usage
-fi
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-Instruments=${InstrSetInstrs[${InstrSet}]}
-if [ "${Instruments}" == "" ]
-then
-    echo "Unrecognized instrument settings: ${InstrSet}"
-    usage
-fi
-
-# -------------------- S e t t i n g s
-RootDir=/home/cvtt/prod
-# ConfigFile="${RootDir}/config/md_recorder.cfg"
-ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
-
-AdminPort="720${InstrSet}"
-Exchange=COINBASE_AT
-DbCredKey=TSDB_COINBS_1
-Name=MD_RECORDER_COINBASE
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/config_server_start.sh

@@ -1,128 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=5
-fi
-
-RootDir=/home/cvtt/prod
-CfgSvcDir=${RootDir}/config_service
-
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Name=config_server
-ServerPy="${RootDir}/cvttpy/apps/utils/config_server.py"
-ServicePort=6789
-ServiceDir=${CfgSvcDir}/data
-LogDir=${RootDir}/logs
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH="${RootDir}:${PYTHONPATH}"
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${ServerPy}"
-Cmd="${Cmd} --port=${ServicePort}"
-Cmd="${Cmd} --root=${ServiceDir}"
-Cmd="${Cmd} --log_file=${LogDir}/%T.config_service.log"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-function check_it {
-    if !(timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${ServicePort}/__shutdown__ ..."
-        timeout 5 curl -s localhost:${ServicePort}/__shutdown__
-
-        pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/__deprecated__/executor_app_start.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-Script="${0} ${*}"
-
-function usage {
-    echo "Usage: ${0}  [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
-    exit 1
-}
-
-for arg in "${@}"
-do
-    case ${arg} in
-        --restart)
-            Restart=Y
-            shift
-            ;;
-        --stop)
-            Stop="Y"
-            shift
-            ;;
-        --once)
-            Once="Y"
-            shift
-            ;;
-        --inst_set=*)
-            InstrSet="${arg#*=}"
-            shift
-            ;;
-        --sleep_sec=*)
-            SleepSec="${arg#*=}"
-            shift
-            ;;
-        -*|--*)
-            usage
-            ;;
-        *)
-            ;;
-    esac
-done
-
-
-if [ "${SleepSec}" == "" ]
-then
-    SleepSec=3
-fi
-
-
-# -------------------- S e t t i n g s
-ConfigServiceHost=cloud11.cvtt.vpn
-ConfigServicePort=6789
-
-RootDir=/home/cvtt/prod
-
-ConfigFile="http://${ConfigServiceHost}:${ConfigServicePort}/apps/executor_app"
-
-AdminPort="7210"
-Exchange=COINBASE_AT
-Name=EXECUTOR_APP
-# -------------------- S e t t i n g s
-
-LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}.log"
-
-source ${HOME}/.pyenv/python3.10-venv/bin/activate
-export PYTHONPATH=${RootDir}
-
-Cmd="nohup"
-Cmd="${Cmd} python3"
-Cmd="${Cmd} ${RootDir}/cvttpy/apps/executor_app.py"
-Cmd="${Cmd} --config=${ConfigFile}"
-Cmd="${Cmd} --active_exchanges=${Exchange}"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_level=INFO"
-Cmd="${Cmd} --log_file=${LogFile}"
-Cmd="${Cmd} &"
-
-function start_it {
-    echo ${Cmd}
-    eval ${Cmd} &
-    sleep 10
-}
-
-
-function check_it {
-
-    if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    then
-        echo "${Script} doesn't respond. restarting..."
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-        start_it
-    fi
-}
-
-function kill_it {
-    pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut  -d' '  -f2)
-
-    if [ "${pids}" != "" ]
-    then
-        echo "Killing ${pids} ..."
-        kill -9 ${pids}
-    fi
-    while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
-    do
-        echo "Shutting down localhost:${AdminPort}/shutdown ..."
-        timeout 5 curl -s localhost:${AdminPort}/shutdown
-        sleep 3
-
-        pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut  -d' '  -f2)
-        echo pids=${pids}
-        if [ "${pids}" != "" ]
-        then
-            kill -9 ${pids}
-        fi
-    done
-}
-
-if [ "${Stop}" == "Y"  ] ; then
-    kill_it
-    exit 0
-fi
-
-if [ "${Once}" == "Y"  ] ; then
-    start_it
-    exit 0
-fi
-
-
-if [ "${Restart}" == "Y"  ]
-then
-    kill_it
-fi
-
-start_it
-while true
-do
-    check_it
-    echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
-    sleep ${SleepSec}
-done
-
-

scripts/healthcheck/cvtt_ts_status_check.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " [-c <config (dflt: apps/cvtt_eqt_alpaca)>]"
+    echo -n " [-s <config_serverice (dflt: http://cloud23.cvtt.vpn:6789)>]"
+    echo -n " [-N <name (dflt: ALPACA-SNDBX)>]"
+    echo
+    exit 1
+}
+
+
+Python="${HOME}/.pyenv/python3.12-venv/bin/python3.12"
+RootDir="${HOME}/prod"
+
+# **** D E B U G
+RootDir=/home/oleg/develop/cvtt2
+# **** D E B U G
+
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel=Status-CVTT
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigService=http://cloud23.cvtt.vpn:6789
+Config=apps/cvtt_eqt_alpaca
+Name="ALPACA-SNDBX"
+
+echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
+
+while getopts ":s:c:n:" opt; do
+    case ${opt} in
+        c )
+            Config=$OPTARG
+            ;;
+        s )
+            ConfigService=$OPTARG
+            ;;
+        n )
+            Name=$OPTARG
+            ;;
+        \? )
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            ;;
+        : )
+            echo "Option -$OPTARG requires an argument." >&2
+            usage
+            ;;
+    esac
+done
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+tmpfile=$(mktemp)
+function cleanup {
+    # echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
+
+
+Cmd="${Python}"
+Cmd="${Cmd} ${RootDir}/cvttpy/apps/utils/services_checker.py"
+Cmd="${Cmd} --config=${ConfigService}/${Config}"
+Cmd="${Cmd} --log_level=ERROR"
+Cmd="${Cmd} --log_stdout"
+Cmd="${Cmd} | tee ${tmpfile} 2>&1"
+
+echo ${Cmd}
+eval ${Cmd}
+
+cat ${tmpfile}
+exit
+
+
+# echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
+# echo  >> ${tmpfile}
+# echo "| host | filesystem | usage |"  >> ${tmpfile}
+# echo "| --- | --- | --- |"  >> ${tmpfile}
+# cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
+# cat ${tmpfile} |  ${Sender} ${StatusChannel}
+

scripts/healthcheck/storage_health_check.sh

@@ -69,7 +69,8 @@ function storage_check() {
         Cmd="ssh -p ${port} $host"
         Cmd="${Cmd} eval \"df -hTl"
         Cmd="${Cmd} -x squashfs"
-        Cmd="${Cmd} | grep -v tmpfs"
+        Cmd="${Cmd} -x tmpfs"
+        Cmd="${Cmd} -x vfat"
         Cmd="${Cmd} | grep -v Filesystem\""
 
         IFS=$'\n' ; lines=$(eval ${Cmd})

scripts/mkt_data/alpaca_hist_md/alpaca_md.sh

@@ -45,7 +45,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 # export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
 
 export PYTHONPATH=/home/cvtt/prod
-export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
 

scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=${HOME}/prod
 
-Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Python=${HOME}/.pyenv/python3.12-venv/bin/python3
 Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
 DbFile=${HOME}/prod/data/rel_liquidity.db
 Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK (cvtt-md.cvtt.vpn)" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK (cryptoval4)" >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+run_checklist  | tee -a ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+
+echo "Sending result to ${Sender} ${StatusChannel}"
+cat ${tmpfile} |  ${Sender} ${StatusChannel}

scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh

@@ -2,7 +2,7 @@
 
 export PYTHONPATH=/home/cvtt/prod
 
-Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3"
 Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
 Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
 Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"

scripts/mkt_data/legacy_eqt_md/process_md.sh

@@ -12,7 +12,7 @@ fi
 rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
 
 # create db files
-PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.12-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
 
 if [ "$?" != "0" ] ; then
     exit

scripts/mkt_data/sync_market_data.sh

@@ -2,7 +2,7 @@
 
 Source=/home/cvtt/prod/archive/md_archive/
 Targets=
-Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
 
 

utils/git/create_remote_repo.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ---------------------
+# user "git" owned remote repository
+#
+# must run on remote server
+# change GitTopDir, HostName and RemoteName
+# ---------------------
+# R e s u l t  E x a m p l e:
+#     sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
+# ---------------------
+#
+
+# ----- S e t t i n g s
+GitTopDir=/ops/store/git
+HostName=cloud21.cvtt.vpn
+RemoteName=origin
+# ----- S e t t i n g s
+
+usage() {
+    echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
+    exit 1
+}
+
+
+RepoName=${1}
+if [ "" == "${RepoName}" ] ; then
+    usage
+fi
+
+RepoParentDir=${GitTopDir}/cvtt2
+if [ "${2}" != "" ] ; then
+    RepoParentDir=${2}
+fi
+
+RepoDir="${RepoParentDir}/${RepoName}.git"
+
+
+Cmd="sudo bash -c 'mkdir ${RepoDir}"
+Cmd="${Cmd} && cd ${RepoDir}"
+Cmd="${Cmd} && git init --bare "
+Cmd="${Cmd} && chown -R git:git ${RepoDir}'"
+echo ${Cmd}
+eval ${Cmd} || exit 1
+
+echo "====================="
+echo "Repository is created. Use this command to add it:"
+echo "     git remote add ${RemoteName} git@${HostName}:${RepoDir}"
+echo "====================="