fix

cronjobs/cvtt/cloud16.cron

@@ -1,10 +0,0 @@
-0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart config_service 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_service.restart.log
-#
-#
-#---------------------- Utils
-0 5 * * * /home/cvtt/prod/utils/config_svc_backup.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_svc_backup.log
-0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_logs.log
-10 1 * * * /home/cvtt/prod/utils/move_archives.sh  2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
-
-#
-#---------------------- T E S T

cronjobs/cvtt/cloud17.cron

@@ -1,11 +0,0 @@
-0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart all 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).bnbspot_md_recorder.restart.log
-#
-#10 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart executor 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).executor_app.restart.log
-#
-#---------------------- Utils
-0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
-10 1 * * * /home/cvtt/prod/utils/move_archives.sh cloud17 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
-#
-# ----------------- #  TEST
-#
-# ----------------- #  TEST

cronjobs/cvtt/cloudstore.cron

@@ -1,8 +0,0 @@
-20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\
-20 6 * * SAT /home/cvtt/prod/run/prune_eqt_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(da
-#
-0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/p
-#
-# T E S T
-#----------------------------------------------------------------------
-#----------------------------------------------------------------------

cronjobs/cvtt/cryptoval3.cron

@@ -1,11 +0,0 @@
-#
-#---------------------- Market Data
-0 6 * * * /home/cvtt/prod/run/archive_yesterday_md.sh  coinbase,bnbspot,bnbfut 2>&1  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_yestd_md.log
-#---------------------- Utils
-0 7 * * * /home/cvtt/prod/utils/backup_md_archive.sh  2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).backup_md_archive.log
-#
-30 7 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
-#
-# ----------------- #  TEST
-#
-# ----------------- #  TEST

cronjobs/save_cronjobs.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: ${0} <local_root_dir>"
+    exit 1
+}
+
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
+    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
+    Cmd="${Cmd} | jq -r"
+    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
+    # echo $Cmd  ### - FOR DEBUG ONLY
+    eval ${Cmd}
+}
+
+Domain=cvtt.vpn
+echo "Started ${0} ${*} ..."
+
+RootDir=${1}
+
+if [ ! -d "${RootDir}" ]; then
+    usage
+fi
+
+cd ${RootDir}
+Cmd="git pull"
+echo ${Cmd} && eval ${Cmd}
+
+for User in cvtt oleg
+do
+    echo "User=$User"
+    Cmd="mkdir -p ${RootDir}/${User}"
+    echo ${Cmd} && eval ${Cmd}
+
+    hosts=$(get_user_hosts ${User} ${Domain})
+    echo ${hosts}
+
+    for host in ${hosts}
+    do
+        echo "===================="
+        echo "${host}:${User}"
+        Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
+        echo ${Cmd} && eval ${Cmd}
+    done
+done
+
+cd ${RootDir}
+
+Cmd="find -type f -empty -delete -print"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="find -type d -empty -delete -print"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git add ."
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git commit -a -m '$(date)'"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git pushall"
+echo ${Cmd} && eval ${Cmd}
+
+echo "Done ${0} ${*}"

docker/alpaca_md_day.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+date_to_load=${1}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_md_day #:latest
+LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
+ContainerName=alpaca_md_day
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+get_prev_business_day() {
+    Start=${1}
+    while true; do
+        if is_business_day ${Start}; then
+            break
+        fi
+        echo "${Start} is not business day in US" >&2
+        Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
+    done
+    echo ${Start}
+}
+export -f get_prev_business_day
+
+if [ -z ${date_to_load} ] ; then
+    echo "Yesterday data has priority. Running historical container will be stopped"
+    Cmd="docker stop ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+    Cmd="docker kill ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+    Cmd="docker rm -f ${ContainerName}"
+    echo ${Cmd} && eval ${Cmd}
+
+    Cmd="docker ps"
+    echo ${Cmd} && eval ${Cmd}
+
+    if is_container_running "$ContainerName"; then
+        echo "Container ${ContainerName} is still running."
+        exit 3
+    fi
+else
+    echo "Historical run ${date_to_load}"
+    if is_container_running "$ContainerName"; then
+        echo "Container ${ContainerName} is already running."
+        exit 3
+    fi
+    if [ "${date_to_load}" == "next" ] ; then
+        if [ ! -e ${LastDayFile} ]; then
+            echo "File ${LastDayFile} does not exist. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+        fi
+        last_date=$(cat ${LastDayFile} | xargs)
+        if [ -z "${last_date}" ] ; then
+            echo "File ${LastDayFile} returned an empty last day. Will try to use prev file."
+            if [ ! -e ${LastDayFile}.prev ]; then
+                echo "File ${LastDayFile}.prev does not exist. Aborted."
+                exit 2
+            fi
+            mv ${LastDayFile}.prev ${LastDayFile}
+            last_date=$(cat ${LastDayFile} | xargs)
+            if [ -z "${last_date}" ] ; then
+                echo "Unable to obtain last_date. Aborted"
+                exit 3
+            fi
+            echo "No last date_to_load in ${LastDayFile}"
+            exit 1
+        fi
+        mv ${LastDayFile} ${LastDayFile}.prev
+        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    fi
+fi
+if [ -n "${date_to_load}" ]; then
+    date_to_load=$(get_prev_business_day ${date_to_load})
+    echo "Historical Data for ${date_to_load}"
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${date_to_load}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi
+
+# truncate to avoid false positive
+date_to_load=$(echo "${date_to_load}" | xargs)
+if [ -n "${date_to_load}" ]; then
+    echo "Saving date_to_load to ${LastDayFile}"
+    echo ${date_to_load} >  ${LastDayFile}
+fi

docker/config_service.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=cvtt_config_service"
+Cmd="${Cmd} -p 6789:6789"
+Cmd="${Cmd} -v /home/cvtt/prod/config_service/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/config_service:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/crypto_md_day.sh

@@ -11,6 +11,7 @@ if [ -z ${date} ] ; then
 fi
 
 Cmd="docker run"
+Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
 Cmd="${Cmd} --rm"

docker/md_recorder.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
+# runs on host to start container
+usage() {
+  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
+  exit 1
+}
+
+is_valid() {
+  local job=$1
+  for valid_job in "${ValidJobs[@]}";
+  do
+    # echo "job=$job valid_job=$valid_job"
+    if [[ "${job}" == "${valid_job}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+
+job=${1}
+if  ! is_valid "${job}"; then
+    usage
+fi
+
+ContainerName="md_recorder.${job}"
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running. Aborted."
+    exit
+fi
+
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
+Cmd="${Cmd} ${job}"
+
+echo ${Cmd}
+eval ${Cmd}

docker/md_recorder_monitor.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+usage() {
+  echo "Usage: $0 <instance (CLD21,CVTTDATA)> [<admin_port (def. 7225)>]"
+  exit 1
+}
+
+ValidInstances=('CLD21' 'CVTTDATA')
+is_valid() {
+  local inst=$1
+  for valid_inst in "${ValidInstances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+is_container_running() {
+    local container_name=$1
+
+    if [ "$(docker ps --filter "name=^/${container_name}$" --filter "status=running" -q)" ]; then
+        return 0  # true
+    else
+        return 1  # false
+    fi
+}
+
+
+
+
+Instance=${1}
+if ! is_valid "${Instance}"; then
+    usage
+fi
+
+AdminPort=7225
+if [ "${2}" != "" ]; then
+    AdminPort=${2}
+fi
+
+ContainerName="md_recorder_monitor.${Instance}"
+
+if is_container_running "$ContainerName"; then
+    echo "Container ${ContainerName} is already running. Aborted."
+    exit
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
+Cmd="${Cmd} ${Instance} ${AdminPort}"
+
+echo ${Cmd}
+eval ${Cmd}

docker/relative_liquidity.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+#Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name=relative_liquidity_svc"
+Cmd="${Cmd} -p 5678:5678"
+Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
+Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
+Cmd="${Cmd} cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+
+echo ${Cmd}
+eval ${Cmd}

docker/trading_calendar.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#Cmd="docker pull apptasticsoftware/trading-calendar:latest"
+#echo ${Cmd} && eval ${Cmd}
+
+Cmd="docker run"
+Cmd="${Cmd} -d"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} --pull=always"
+Cmd="${Cmd} --name trading-calendar"
+Cmd="${Cmd} -p 8000:80 apptasticsoftware/trading-calendar"
+echo ${Cmd} && eval ${Cmd}

release_version.txt

@@ -1 +1 @@
-1.0.3
+1.3.6

research/load_crypto_md.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
+  exit 1
+}
+
+is_valid() {
+  local inst=$1
+  local valid_instances=$2
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+date=""
+host=homestore
+source=cloud21
+
+while getopts ":h:d:s:" opt; do
+  case ${opt} in
+    d )
+      date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      source=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+echo "$date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo h
+    usage
+fi
+
+valid_sources=('cloud21' 'cvttdata')
+if ! is_valid ${source} ${valid_sources[@]} ; then
+    echo s
+    usage
+fi
+
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=oleg
+    SourceRootDir=/works/cvtt/md_archive/crypto/${source}
+else
+    usage
+fi
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir}"
+
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file

scripts/alpaca_hist_md/alpaca_md_day.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-date_to_load=${1}
-
-DockerRegistry=cloud21.cvtt.vpn:5500
-DockerImage=${DockerRegistry}/alpaca_md_day:latest
-LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
-
-if [ -z ${date_to_load} ] ; then
-    echo "Yesterday data. Historical container will be killed"
-    docker kill alpaca_md_day
-else
-    if [ "${date_to_load}" == "next" ] ; then
-        last_date=$(cat ${LastDayFile})
-        if [ -z ${last_date} ] ; then
-            echo "No last date_to_load in ${LastDayFile}"
-            exit 1
-        fi
-        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
-    fi
-    echo "Historical Data for ${date_to_load}"
-fi
-
-Cmd="docker run"
-Cmd="${Cmd} --network=host"
-Cmd="${Cmd} --name=alpaca_md_day"
-Cmd="${Cmd} --rm"
-Cmd="${Cmd} ${DockerImage}"
-Cmd="${Cmd} ${date_to_load}"
-
-echo $Cmd
-eval $Cmd
-
-if [ "$?" != "0" ] ; then
-    exit 1 # if killed we do not save last day
-fi
-
-if [ -n ${date_to_load}]; then
-    echo "Saving date_to_load to ${LastDayFile}"
-    echo ${date_to_load} >  ${LastDayFile}
-fi

scripts/alpaca_hist_md/load_rel_liq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export PYTHONPATH=${HOME}/prod
+
+Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
+DbFile=${HOME}/prod/data/rel_liquidity.db
+Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
+
+Cmd="${Python}"
+Cmd="${Cmd} ${Script}"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --db_file=${DbFile}"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo Done ${0} ${*}

scripts/build_release.sh

@@ -7,25 +7,46 @@ git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
 git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 
-default_project=cvttpy
-default_branch=master
-
-
 dist_root=/home/cvttdist/software/cvtt2
 dist_user=cvttdist
 dist_host="cloud21.cvtt.vpn"
 dist_ssh_port="22"
 
 dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+# ---------------- Settings
 
-interactive=Y
+# ---------------- cmdline
-
+prj=
-# cmdline
+brnch=master
-prj=${1:-${default_project}}
-brnch=${2:-${default_branch}}
-if [ "${3}" == "D" ]; then
 interactive=N
-fi
+
+usage() {
+  echo "Usage: $0 -p <project> [-b <branch (master)> -i (interactive)"
+  exit 1
+}
+
+while getopts ":p:b:i" opt; do
+  case ${opt} in
+    p )
+      prj=$OPTARG
+      ;;
+    b )
+      brnch=$OPTARG
+      ;;
+    i )
+      interactive=Y
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
 
 function confirm {
     if [ "${interactive}" == "Y" ]; then

scripts/crypto_hist_md/exchanges_trading_stats.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PYTHONPATH=/home/cvtt/prod
+
+Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
+Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
+Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
+Cmd="${Cmd} --instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD"
+Cmd="${Cmd} --db_file=/home/cvtt/prod/data/exchange_trading_stats.db"
+Cmd="${Cmd} --log_file=/home/cvtt/prod/logs/%%T.EXCHANGE_TRADING_STATS.log"
+
+echo ${Cmd}
+eval ${Cmd}

scripts/healthcheck/storage_health_check.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
 
 RootDir="${HOME}/prod"
@@ -10,32 +10,24 @@ StatusChannel=Status-CVTT
 AlertChannel=Alerts-CVTT
 Sender=${RootDir}/ops/utils/send_mmost.sh
 
+# ----- For DEBUGGING
 # Sender=cat
 # StatusChannel=
 
-Hosts=
+get_user_hosts() {
-Hosts="${Hosts} cloud11.cvtt.vpn"
+    local User=${1}
-Hosts="${Hosts} cloud15.cvtt.vpn"
+    local Domain=${2}
-Hosts="${Hosts} cloud16.cvtt.vpn"
-Hosts="${Hosts} cloud17.cvtt.vpn"
-Hosts="${Hosts} cloud21.cvtt.vpn"
-Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
-Hosts="${Hosts} cvttdata.cvtt.vpn"
+    Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
-Hosts="${Hosts} cryptoval2.cvtt.vpn"
+    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
-Hosts="${Hosts} cryptoval3.cvtt.vpn"
+    Cmd="${Cmd} | jq -r"
+    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
 
-Hosts="${Hosts} homestore.cvtt.vpn"
+    Cmd="${Cmd} | sed  's/\$/.${Domain}/'"
-Hosts="${Hosts} nsbackup.sheynin.home"
+    eval ${Cmd}
-Hosts="${Hosts} dtvmhost.cvtt.vpn"
+}
-Hosts="${Hosts} ops-server.cvtt.vpn"
-
-Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
-Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
-Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
-
-Metrics=()
-TempFiles=
 
 function cleanup {
     echo Cleaing up temporary files: ${TempFiles}
@@ -61,12 +53,14 @@ function space_alert() {
 }
 
 function storage_check() {
+    local Hosts=("${@}")
 
     result_lines=()
 
     declare -a SingleMeas
-    for host in ${Hosts}
+    for host in ${Hosts[@]}
     do
+        echo "storage_check host=${host}" >&2
         if [[ "${host}" == *"cryptovaltrading.com" ]]; then
             port=7822
         else
@@ -98,13 +92,22 @@ function storage_check() {
     done
 }
 
+User=oleg
+Metrics=()
+TempFiles=
+
+Hosts=()
+for Domain in cvtt.vpn cryptovaltrading.com ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
 tmpfile=$(mktemp)
 TempFiles="${TempFiles} ${tmpfile}"
 
 tmpfile2=$(mktemp)
 TempFiles="${TempFiles} ${tmpfile2}"
 
-storage_check > ${tmpfile2}
+storage_check  "${Hosts[@]}" > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
 echo  >> ${tmpfile}