load EQT md to GPUshnik fix

release_version.txt

@@ -1 +1 @@
-1.4.8F,mostly cleaning+fix
+1.5.2,load EQT md to GPUshnik fix

research/load_eqty_md.sh

@@ -90,7 +90,6 @@ TargetFilePath="${TargetDir}/${TargetFile}"
 mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
 touch ${TargetFilePath}
 
-
 # ---- temp dir
 echo ${SourceFile}
 tmp_dir=$(mktemp -d)
@@ -136,12 +135,19 @@ function download_file {
     Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
     echo ${Cmd}
     eval ${Cmd}
+
+    if [[ ! -f ${tmp_dir}/${SourceFileZip} ]] ; then
+      echo "File ${SourceUser}@${SourceHost}:${SourceFilePath} is missing. Skipped."
+      return
+    fi
+
     ls -lh ${tmp_dir}
     Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
     echo ${Cmd} && eval ${Cmd}
 
     DbFile=${tmp_dir}/${SourceFile}
 
+
 # --- T E M P - for files older than 20240813
     sqlite3 ${DbFile} <<EOF
 DROP INDEX md_1min_bars_idx;
@@ -149,7 +155,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, excha
 EOF
 # --- T E M P
 
-    Cmd="sqlite3 ${DbFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+    Cmd="sqlite3 ${DbFile} \".dump\" | sqlite3 ${TargetFilePath}"
     echo ${Cmd}
     eval ${Cmd}
 

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+
+
+exit
+# homestore
+echo homestore
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo cloud21
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo "------ gpushnik"
+ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'

utils/git/create_remote_repo.sh

@@ -4,7 +4,7 @@
 # user "git" owned remote repository
 #
 # must run on remote server
-# change HostName and RemoteName
+# change GitTopDir, HostName and RemoteName
 # ---------------------
 # R e s u l t  E x a m p l e:
 #     sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
@@ -12,12 +12,13 @@
 #
 
 # ----- S e t t i n g s
+GitTopDir=/ops/store/git
 HostName=cloud21.cvtt.vpn
 RemoteName=origin
 # ----- S e t t i n g s
 
 usage() {
-    echo "Usage: ${0} <repo_name> [<repo_parent_dir> */opt/store/git/cvtt2]"
+    echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
     exit 1
 }
 
@@ -27,7 +28,7 @@ if [ "" == "${RepoName}" ] ; then
     usage
 fi
 
-RepoParentDir=/opt/store/git/cvtt2
+RepoParentDir=${GitTopDir}/cvtt2
 if [ "${2}" != "" ] ; then
     RepoParentDir=${2}
 fi