prepare crypto sim enhanced

VERSION

@@ -1 +1 @@
-2.0.4,replacing "latest" with real docker image tag
+2.0.8,prepare crypto sim enhanced

docker/market_data/hist/crypto_md_day.sh

@@ -1,18 +1,30 @@
 #!/bin/bash
+usage() {
+    echo "Usage: ${0} <docker_image_tag> <host> [<date>]"
+    exit 1
+}
 
 # ------- Settings
-ImageTag=1.6.9
 DockerRegistry=cloud21.cvtt.vpn:5500
-DockerImage=${DockerRegistry}/crypto_md_day:${ImageTag}
 # ------- Settings
 
-host=${1}
-date=${2}
+ImageTag=${1}
+host=${2}
+date=${3}
+
+if [ -z ${ImageTag} ] ; then
+    usage
+fi
+
+if [ -z ${host} ] ; then
+    usage
+fi
 
 if [ -z ${date} ] ; then
     date=$(date -d "yesterday" +%Y%m%d)
 fi
 
+DockerImage=${DockerRegistry}/crypto_md_day:${ImageTag}
 
 
 Cmd="docker run"

docker/market_data/hist/load_alpaca_hbar.sh

@@ -75,7 +75,7 @@ while getopts ":d:N:L:t:" opt; do
   esac
 done
 
-DockerImage=${DockerRegistry}/alpaca_hbar/${ImageTag} #:latest
+DockerImage=${DockerRegistry}/alpaca_hbar:${ImageTag} #:latest
 
 
 if is_container_running "$ContainerName"; then

research/prepare_crypto_simdata.sh

@@ -10,17 +10,22 @@
 
 
 usage() {
-  echo "Usage: $0  -d YYYMMDD Date> [-O <output dir (./) >]"
+  echo -n "Usage: $0"
+  echo -n " -d YYYMMDD Date>"
+  echo -n " [-O <output dir (./) >]"
+  echo -n " [-S <archive_source_host (cloud21)>]"
+  echo -n " [-D <orig_db_source> (cloud21)]"
+  echo
   exit 1
 }
 
 # --------------------- Settings
-SourceHost=cloud21.cvtt.vpn
+SourceHost=cloud21
 SourceRootDir=/opt/store/cvtt/md_archive/crypto
 DbSource=cloud21
 # --------------------- Settings
 
-while getopts ":d:O:" opt; do
+while getopts ":d:O:S:D:" opt; do
   case ${opt} in
     d )
       Date=$OPTARG
@@ -28,6 +33,12 @@ while getopts ":d:O:" opt; do
     O )
       OutputDir=$OPTARG
       ;;
+    S )
+      SourceHost=$OPTARG
+      ;;
+    D )
+      DbSource=$OPTARG
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -46,6 +57,8 @@ fi
 if [ -z ${OutputDir}  ] ; then
     OutputDir=.
 fi
+
+SourceHost+=".cvtt.vpn"
 mkdir -p ${OutputDir}
 
 # --- Binance

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -27,8 +27,10 @@ run_checklist() {
     Commands=(
         ["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
         ["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
-        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
-        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5"
         ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
     )
 

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -29,12 +29,14 @@ run_checklist() {
         Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
     done
     Commands["hs01"]+="echo"
+    Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" 
 
     Commands["cloud21"]=""
     for sym in ${CheckSymbols}; do
-        Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+        Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
     done
     Commands["cloud21"]+="echo"
+    Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" 
 
     Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
 
@@ -46,6 +48,13 @@ run_checklist() {
     done
 }
 
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
 # run_checklist
 
 tmpfile=$(mktemp)