progress

release_version.txt

@@ -1 +1 @@
-0.6.7
+0.7.9

scripts/alpaca_hist_md/alpaca_md.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
 Start=${1}
-End=${2}
+NumJobs=${2}
-NumJobs=${3}
+InstListFile=${3}
-InstListFile=${4}
 
 export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
 
@@ -30,25 +29,36 @@ if ! is_business_day ${Start}; then
     exit
 fi
 
-if [ "${End}" == "" ] ; then
 End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
+
+# Exclude today
+if [ "${End}" == $(date '+%Y-%m-%d') ] ; then
+    End=$(date -d 'yesterday' ''+%Y-%m-%d'')
 fi
 
 if [ "${NumJobs}" == "" ] ; then
-    NumJobs=25
+    NumJobs=30
 fi
 
 echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
 
+# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
+
 export PYTHONPATH=/home/cvtt/prod
 export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
-export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
-
-# export OutputDir=/home/cvtt/alpaca_md
-export OutputDir=/home/cvtt/host_drive/alpaca_md
 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
 export LogDir=/home/cvtt/prod/logs/alpaca_md
 
+# ----- T E M P
+# export PYTHONPATH=/home/oleg/develop/cvtt2
+# export Python=/home/oleg/.pyenv/python3.10-venv/bin/python3
+# export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+# export PyScript=/home/oleg/develop/cvtt2/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+# export OutputDir=/home/oleg/develop/cvtt2/tmp # Local
+# export LogDir=/home/oleg/develop/cvtt2/tmp
+
 mkdir -p ${LogDir}
 
 run_proc() {
@@ -59,20 +69,21 @@ run_proc() {
 
     Cmd="${Python} ${PyScript}"
     Cmd="${Cmd} --config=${Config}"
-    Cmd="${Cmd} --output_dir=${OutputDir}"
+    Cmd="${Cmd} --output_dir=${OutputDir}/${Start:0:4}"
     Cmd="${Cmd} --instruments=ALPACA:${Inst}"
     Cmd="${Cmd} --start=${Start}"
     if [ "${End}" != "" ]; then
         Cmd="${Cmd} --end=${End}"
     fi
-    Cmd="${Cmd} --log_file=${LogDir}/${Start}.${Inst}.log"
+    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
+    Cmd="${Cmd} --log_level=WARNING"
     echo ${Cmd}
     eval ${Cmd}
 }
 
 export -f run_proc
 
-if [ -n $InstListFile ]; then
+if [ "$InstListFile" == "" ]; then
 
     key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
     secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
@@ -95,7 +106,7 @@ do
     Instruments+=("$Inst")
 done
 
-slice_size=100 # 10K symbols parallel cannot handle
+slice_size=500 # 10K symbols parallel cannot handle
 for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
     InstSlice=("${Instruments[@]:ii:slice_size}")
 

scripts/alpaca_hist_md/run_alpaca_hist_md.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+

scripts/build_release.sh

@@ -1,10 +1,11 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # ---------------- Settings
 declare -A git_repo_arr
 
 git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
+git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 
 default_project=cvttpy
 default_branch=master
@@ -63,6 +64,13 @@ else
 fi
 
 tmp_dir=$(mktemp -d)
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+
 prj_dir="${tmp_dir}/${prj}"
 
 cmd_arr=()
@@ -149,7 +157,4 @@ do
     pwd && echo ${cmd} && eval ${cmd}
 done
 
-cd ${HOME}
+echo "$0 Done ${project} ${release_version}"
-rm -rf ${tmp_dir}
-
-echo $0 Done

scripts/crypto_hist_md/process_daily_md.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+date=${1}
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+SourceHost=cryptoval3.cvtt.vpn
+SourceUser=cvtt
+SourceRootDir=/localdisk/cvtt/archive/md_archive
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then    
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file

scripts/healthcheck/storage_health_check.sh

@@ -15,7 +15,6 @@ Sender=${RootDir}/ops/utils/send_mmost.sh
 
 Hosts=
 Hosts="${Hosts} cloud11.cvtt.vpn"
-Hosts="${Hosts} cloud14.cvtt.vpn"
 Hosts="${Hosts} cloud15.cvtt.vpn"
 Hosts="${Hosts} cloud16.cvtt.vpn"
 Hosts="${Hosts} cloud17.cvtt.vpn"
@@ -23,17 +22,17 @@ Hosts="${Hosts} cloud21.cvtt.vpn"
 Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
 Hosts="${Hosts} cvttdata.cvtt.vpn"
-Hosts="${Hosts} cryptoval1.cvtt.vpn"
 Hosts="${Hosts} cryptoval2.cvtt.vpn"
 Hosts="${Hosts} cryptoval3.cvtt.vpn"
 
 Hosts="${Hosts} homestore.cvtt.vpn"
-Hosts="${Hosts} nsbackup.cvtt.vpn"
+Hosts="${Hosts} nsbackup.sheynin.home"
 Hosts="${Hosts} dtvmhost.cvtt.vpn"
 Hosts="${Hosts} ops-server.cvtt.vpn"
 
 Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
 Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
+Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
 
 function space_alert() {
     ALERT_USAGE=75%

scripts/set_version.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 
+
+# ----- Settings
+LocalSoftwareDir=${HOME}/software/cvtt2
+
+ProdDir=${HOME}/prod
+
+ReleaseHosts=("cloud21.cvtt.vpn")
+ReleasePorts=("22")
+ReleaseUsers=("cvttdist")
+ReleaseDir=("/home/cvttdist/software/cvtt2")
+# ----- Settings
+
 function usage() {
     echo "Usage: ${0} <project> <version>"
     exit 1
@@ -18,17 +30,6 @@ then
     Version=latest
 fi
 
-# ----- Settings
-LocalSoftwareDir=${HOME}/software/cvtt2
-
-ProdDir=${HOME}/prod
-
-ReleaseHosts=("cloud21.cvtt.vpn")
-ReleasePorts=("22")
-ReleaseUsers=("cvttdist")
-ReleaseDir=("/home/cvttdist/software/cvtt2")
-# ----- Settings
-
 function rsync_load_version() {
     for idx in "${!ReleaseHosts[@]}"
     do
@@ -68,6 +69,7 @@ function rsync_load_version() {
 }
 
 mkdir -p ${LocalSoftwareDir}
+mkdir -p ${ProdDir}
 
 # exists and not empty
 rsync_load_version

utils/archive_logs.sh

@@ -26,7 +26,7 @@ Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d')
 
 echo "Looking for log files older than ${DateCriteria} in ${LogDir}"
 
-LogArchiveDir=/home/cvtt/prod/archive/logs
+LogArchiveDir=${HOME}/prod/archive/logs
 mkdir -p ${LogArchiveDir}
 
 echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')'  -type f -not -newermt ${Oldest})"