fix

release_version.txt

@@ -1 +1 @@
-0.7.7
+0.8.7

scripts/alpaca_hist_md/alpaca_md.sh

@@ -48,7 +48,7 @@ export PYTHONPATH=/home/cvtt/prod
 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
 export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
-export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
+export OutputDir=/home/cvtt/prod/archive/equities/alpaca_md # Local
 export LogDir=/home/cvtt/prod/logs/alpaca_md
 
 # ----- T E M P

scripts/archive_yesterday_md.sh

@@ -4,14 +4,14 @@ Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 
-host=$(hostname)
+host=${1}
 if [ "${host}" == "cvttdata" ]
 then
-    ArchiveRootDir=/home/cvtt/prod/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/cvttdata
     CredKey=TSDB_MD_CVTTDATA_RO
-elif [ "${host}" == "cloud21.cryptovaltrading.com" ]
+elif [ "${host}" == "cloud21" ]
 then
-    ArchiveRootDir=/opt/store/cvtt/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/cloud21
     CredKey=TSDB_MD_CLD21_RO
 else
     echo "Unknown host ${host}. ${0} Aborted."
@@ -21,7 +21,7 @@ fi
 mkdir -p ${ArchiveRootDir}
 
 yesterday=$(date -d "yesterday" +%Y%m%d)
-Schemas=${1}
+Schemas=${2}
 if [ "${Schemas}" == "" ]
 then
     Schemas="coinbase,bnbspot,bnbfut"

scripts/build_release.sh

@@ -64,6 +64,13 @@ else
 fi
 
 tmp_dir=$(mktemp -d)
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+
 prj_dir="${tmp_dir}/${prj}"
 
 cmd_arr=()
@@ -150,7 +157,4 @@ do
     pwd && echo ${cmd} && eval ${cmd}
 done
 
-cd ${HOME}
-rm -rf ${tmp_dir}
-
 echo "$0 Done ${project} ${release_version}"

scripts/healthcheck/storage_health_check.sh

@@ -15,7 +15,6 @@ Sender=${RootDir}/ops/utils/send_mmost.sh
 
 Hosts=
 Hosts="${Hosts} cloud11.cvtt.vpn"
-Hosts="${Hosts} cloud14.cvtt.vpn"
 Hosts="${Hosts} cloud15.cvtt.vpn"
 Hosts="${Hosts} cloud16.cvtt.vpn"
 Hosts="${Hosts} cloud17.cvtt.vpn"
@@ -23,26 +22,38 @@ Hosts="${Hosts} cloud21.cvtt.vpn"
 Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
 Hosts="${Hosts} cvttdata.cvtt.vpn"
-Hosts="${Hosts} cryptoval1.cvtt.vpn"
 Hosts="${Hosts} cryptoval2.cvtt.vpn"
 Hosts="${Hosts} cryptoval3.cvtt.vpn"
 
 Hosts="${Hosts} homestore.cvtt.vpn"
-Hosts="${Hosts} nsbackup.cvtt.vpn"
+Hosts="${Hosts} nsbackup.sheynin.home"
 Hosts="${Hosts} dtvmhost.cvtt.vpn"
 Hosts="${Hosts} ops-server.cvtt.vpn"
 
 Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
 Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
+Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
+
+Metrics=()
+TempFiles=
+
+function cleanup {
+    echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
 
 function space_alert() {
     ALERT_USAGE=75%
-    for ln in "${Measurements[@]}"
+    for metric in "${Metrics[@]}"
     do
-        IFS=$' '; args=($ln); unset IFS
+        IFS=$' '; args=($metric); unset IFS
         host=${args[0]}
         fs=${args[1]}
         space_used=${args[2]}
+
         if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
             echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
         fi
@@ -53,6 +64,7 @@ function storage_check() {
 
     result_lines=()
 
+    declare -a SingleMeas
     for host in ${Hosts}
     do
         if [[ "${host}" == *"cryptovaltrading.com" ]]; then
@@ -71,7 +83,12 @@ function storage_check() {
         do
             IFS=$' '; args=($ln); unset IFS
             res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
+
             result_lines+=("$res")
+
+            fs=${args[6]}
+            used=${args[5]}
+            Metrics+=("$host $fs $used")
         done
         unset IFS
     done
@@ -82,7 +99,11 @@ function storage_check() {
 }
 
 tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 tmpfile2=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile2}"
+
 storage_check > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
@@ -91,17 +112,15 @@ echo "| host | filesystem | usage |"  >> ${tmpfile}
 echo "| --- | --- | --- |"  >> ${tmpfile}
 cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
-rm ${tmpfile} ${tmpfile2}
 
-Measurements=()
 tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 space_alert > ${tmpfile}
-cat ${tmpfile}
 if [ -s ${tmpfile} ]
 then
     (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
 else
-    echo File ${tmpfile} is empty
+    echo "No Storage Alerts"
 fi
-rm ${tmpfile}
 

scripts/set_version.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+
+# ----- Settings
+LocalSoftwareDir=${HOME}/software/cvtt2
+
+ProdDir=${HOME}/prod
+
+ReleaseHosts=("cloud21.cvtt.vpn")
+ReleasePorts=("22")
+ReleaseUsers=("cvttdist")
+ReleaseDir=("/home/cvttdist/software/cvtt2")
+# ----- Settings
+
 function usage() {
     echo "Usage: ${0} <project> <version>"
     exit 1
@@ -18,17 +30,6 @@ then
     Version=latest
 fi
 
-# ----- Settings
-LocalSoftwareDir=${HOME}/software/cvtt2
-
-ProdDir=${HOME}/prod
-
-ReleaseHosts=("cloud21.cvtt.vpn")
-ReleasePorts=("22")
-ReleaseUsers=("cvttdist")
-ReleaseDir=("/home/cvttdist/software/cvtt2")
-# ----- Settings
-
 function rsync_load_version() {
     for idx in "${!ReleaseHosts[@]}"
     do
@@ -68,6 +69,7 @@ function rsync_load_version() {
 }
 
 mkdir -p ${LocalSoftwareDir}
+mkdir -p ${ProdDir}
 
 # exists and not empty
 rsync_load_version