debugging crypto_md_day

cronjobs/save_cronjobs.sh

@@ -1,27 +1,27 @@
-#!/bin/sh
+#!/bin/bash
 
 usage() {
     echo "Usage: ${0} <local_root_dir>"
     exit 1
 }
 
-get_hosts_json() {
-    curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts | ${HOME}/bin/hjson -j
-}
-
 get_user_hosts() {
-    User=${1}
+    local User=${1}
-    Domain=${2}
+    local Domain=${2}
 
-    get_hosts_json | jq -r \
+    Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
-        --arg domain "${Domain}" \
+    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
-        --arg usr "${User}" \
+    Cmd="${Cmd} | jq -r"
-        '.[$domain] | to_entries[] | select(.value.users[] | contains($usr)) | .key'
+    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
+    # echo $Cmd  ### - FOR DEBUG ONLY
+    eval ${Cmd}
 }
 
-echo "Started ${0} ${*} ..."
-
 Domain=cvtt.vpn
+echo "Started ${0} ${*} ..."
+
 RootDir=${1}
 
 if [ ! -d "${RootDir}" ]; then
@@ -35,10 +35,12 @@ echo ${Cmd} && eval ${Cmd}
 for User in cvtt oleg
 do
     echo "User=$User"
+    Cmd="mkdir -p ${RootDir}/${User}"
+    echo ${Cmd} && eval ${Cmd}
+
     hosts=$(get_user_hosts ${User} ${Domain})
-    mkdir -p ${RootDir}/${User}
+    echo ${hosts}
-    print ${hosts}
+
-    exit
     for host in ${hosts}
     do
         echo "===================="

docker/crypto_md_day.sh

@@ -14,11 +14,12 @@ Cmd="docker run"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
-Cmd="${Cmd} --rm"
+# Cmd="${Cmd} --rm"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"
 Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+# Cmd="${Cmd} -s coinbase,bnbspot"
 
 echo $Cmd
 eval $Cmd

release_version.txt

@@ -1 +1 @@
-1.3.2
+1.3.9,debugging crypto_md_day

research/load_crypto_md.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
+  exit 1
+}
+
+is_valid() {
+  local inst=$1
+  shift
+  local valid_instances=("$@")
+
+  for valid_inst in "${valid_instances[@]}";
+  do
+    if [[ "$inst" == "$valid_inst" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+date=""
+host=homestore
+source=cloud21
+
+while getopts ":h:d:s:" opt; do
+  case ${opt} in
+    d )
+      date=$OPTARG
+      ;;
+    h )
+      host=$OPTARG
+      ;;
+    s )
+      source=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+echo "$date $host $source"
+
+valid_hosts=('homestore' 'cloud21')
+if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
+    echo "Host '${host}' is not valid"
+    usage
+fi
+
+valid_sources=('cloud21' 'cvttdata')
+if ! is_valid ${source} ${valid_sources[@]} ; then
+    echo "Source '${source}' is not valid"
+    usage
+fi
+
+if [ "${host}" == "cloud21" ] ; then
+    SourceHost=cloud21.cvtt.vpn
+    SourceUser=cvtt
+    SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
+elif [ "${host}" == "homestore" ]; then
+    SourceHost=homestore.cvtt.vpn
+    SourceUser=oleg
+    SourceRootDir=/works/cvtt/md_archive/crypto/${source}
+else
+    usage
+fi
+
+echo "${SourceHost} ${SourceUser} ${SourceRootDir}"
+
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file

scripts/build_release.sh

@@ -127,7 +127,8 @@ if [ "${?}" != "0" ]; then
 fi
 
 
-release_version=$(cat release_version.txt)
+release_version=$(cat release_version.txt | awk -F',' '{print $1}')
+whats_new=$(cat release_version.txt | awk -F',' '{print $2}')
 
 
 echo "--------------------------------"
@@ -136,7 +137,7 @@ echo "Release version: ${release_version}"
 confirm
 
 version_tag="v${release_version}"
-version_comment="${version_tag}_${project}_${branch}_$(date +%Y%m%d)"
+version_comment="'${version_tag} ${project} ${branch} $(date +%Y-%m-%d)\n${whats_new}'"
 
 cmd_arr=()
 Cmd="git tag -a ${version_tag} -m ${version_comment}"
@@ -178,4 +179,5 @@ do
     pwd && echo ${cmd} && eval ${cmd}
 done
 
+echo "PROJECT ${project} **** P U L L   N E W   T A G S  F R O M  o r i g i n ****"
 echo "$0 Done ${project} ${release_version}"

scripts/healthcheck/storage_health_check.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
 
 RootDir="${HOME}/prod"
@@ -10,32 +10,24 @@ StatusChannel=Status-CVTT
 AlertChannel=Alerts-CVTT
 Sender=${RootDir}/ops/utils/send_mmost.sh
 
+# ----- For DEBUGGING
 # Sender=cat
 # StatusChannel=
 
-Hosts=
+get_user_hosts() {
-Hosts="${Hosts} cloud11.cvtt.vpn"
+    local User=${1}
-Hosts="${Hosts} cloud15.cvtt.vpn"
+    local Domain=${2}
-Hosts="${Hosts} cloud16.cvtt.vpn"
-Hosts="${Hosts} cloud17.cvtt.vpn"
-Hosts="${Hosts} cloud21.cvtt.vpn"
-Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
-Hosts="${Hosts} cvttdata.cvtt.vpn"
+    Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
-Hosts="${Hosts} cryptoval2.cvtt.vpn"
+    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
-Hosts="${Hosts} cryptoval3.cvtt.vpn"
+    Cmd="${Cmd} | jq -r"
+    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
 
-Hosts="${Hosts} homestore.cvtt.vpn"
+    Cmd="${Cmd} | sed  's/\$/.${Domain}/'"
-Hosts="${Hosts} nsbackup.sheynin.home"
+    eval ${Cmd}
-Hosts="${Hosts} dtvmhost.cvtt.vpn"
+}
-Hosts="${Hosts} ops-server.cvtt.vpn"
-
-Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
-Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
-Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
-
-Metrics=()
-TempFiles=
 
 function cleanup {
     echo Cleaing up temporary files: ${TempFiles}
@@ -61,12 +53,14 @@ function space_alert() {
 }
 
 function storage_check() {
+    local Hosts=("${@}")
 
     result_lines=()
 
     declare -a SingleMeas
-    for host in ${Hosts}
+    for host in ${Hosts[@]}
     do
+        echo "storage_check host=${host}" >&2
         if [[ "${host}" == *"cryptovaltrading.com" ]]; then
             port=7822
         else
@@ -98,13 +92,22 @@ function storage_check() {
     done
 }
 
+User=oleg
+Metrics=()
+TempFiles=
+
+Hosts=()
+for Domain in cvtt.vpn cryptovaltrading.com ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
 tmpfile=$(mktemp)
 TempFiles="${TempFiles} ${tmpfile}"
 
 tmpfile2=$(mktemp)
 TempFiles="${TempFiles} ${tmpfile2}"
 
-storage_check > ${tmpfile2}
+storage_check  "${Hosts[@]}" > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
 echo  >> ${tmpfile}