fix

configs/cloud11/supervisord.conf

@@ -1,47 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[program:bnbfut_rec_cv1]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBFUT
-        --instruments=PERP-BTC-USDT,PERP-ETH-USDT
-        --db_credentials_key=TSDB_MD_CV1
-        --admin_port=7226
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CV1.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:coinbase_rec_cv1]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=COINBASE_AT
-        --instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-        --db_credentials_key=TSDB_MD_CV1
-        --admin_port=7224
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CV1.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-

configs/cloud16/supervisor.conf

@@ -1,28 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[program:config_service]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
-        -u /home/cvtt/prod/cvttpy/apps/utils/config_server.py
-        --port=6789
-        --root=/home/cvtt/prod/config_service/data
-        --log_file=/home/cvtt/prod/logs/%%T.CONFIG_SERVICE.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-

configs/cloud17/supervisord.conf

@@ -1,99 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-# [program:executor]
-# directory=/home/cvtt/prod
-# command=/home/cvtt/.pyenv/python3.10-venv/bin/python3
-#         -u /home/cvtt/prod/cvttpy/apps/executor_app.py
-#         --config=http://cloud16.cvtt.vpn:6789/apps/cvtt
-#         --active_exchanges=COINBASE_AT
-#         --admin_port=7220
-#         --log_level=INFO
-#         --log_file=/home/cvtt/prod/logs/%%T.EXECUTOR_APP.log
-# autostart=true
-# autorestart=true
-# startretries=1
-# startsecs=3
-# user=cvtt
-
-#[group:bnbspot_mdrec]
-#programs=
-#    bnbspot_rec_cloud17,
-#    bnbspot_rec_cv1
-
-[program:bnbfut_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBFUT
-        --instruments=PERP-BTC-USDT,PERP-ETH-USDT
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7201
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:coinbase_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=COINBASE_AT
-        --instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7202
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:bnbspot_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBSPOT
-        --instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7203
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-
-[program:bnbspot_rec_cvttdata]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBSPOT
-        --instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-        --db_credentials_key=TSDB_MD_CVTTDATA
-        --admin_port=7222
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CVTTDATA.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt

release_version.txt

@@ -1 +1 @@
-0.5.8
+0.6.9

scripts/alpaca_hist_md/alpaca_assets.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+CredsFile=${1}
+CredKey=${2}
+
+if [ "${CredsFile}" == "" ] ; then
+    CredsFile="${HOME}/.creds"
+fi
+
+if [ "${CredKey}" == "" ] ; then
+    CredKey="ALPACA_SANDBOX"
+fi
+
+key=$(jq -r ".[\"${CredKey}\"] | .api_key" ${CredsFile})
+secret=$(jq -r ".[\"${CredKey}\"] | .secret_key" ${CredsFile})
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+
+eval ${Cmd}
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done

scripts/alpaca_hist_md/alpaca_md.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+Start=${1}
+NumJobs=${2}
+InstListFile=${3}
+
+export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+if [ "${Start}" == "" ] ; then
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+fi
+
+if ! is_business_day ${Start}; then
+    echo "${Start} is not business day in US"
+    exit
+fi
+
+End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
+
+if [ "${NumJobs}" == "" ] ; then
+    NumJobs=30
+fi
+
+echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
+
+export PYTHONPATH=/home/cvtt/prod
+export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
+
+export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export LogDir=/home/cvtt/prod/logs/alpaca_md
+
+mkdir -p ${LogDir}
+
+run_proc() {
+    Inst=${1}
+    Start=${2}
+    End=${3}
+    echo "Running for $Inst"
+
+    Cmd="${Python} ${PyScript}"
+    Cmd="${Cmd} --config=${Config}"
+    Cmd="${Cmd} --output_dir=${OutputDir}"
+    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
+    Cmd="${Cmd} --start=${Start}"
+    if [ "${End}" != "" ]; then
+        Cmd="${Cmd} --end=${End}"
+    fi
+    Cmd="${Cmd} --log_file=${LogDir}/${Start}.${Inst}.log"
+    echo ${Cmd}
+    eval ${Cmd}
+}
+
+export -f run_proc
+
+if [ -n $InstListFile ]; then
+
+    key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
+    secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
+
+    Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+    Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+    Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+    Cmd="${Cmd} --header 'accept: application/json'"
+    Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
+    Cmd="${Cmd}  | sed 's/\"//g'"
+    Cmd="${Cmd} | sed 's/^/STOCK-/'"
+else
+    Cmd="cat ${InstListFile}"
+fi
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
+
+slice_size=500 # 10K symbols parallel cannot handle
+for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
+    InstSlice=("${Instruments[@]:ii:slice_size}")
+
+    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
+done

scripts/alpaca_hist_md/run_alpaca_hist_md.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+

scripts/build_release.sh

@@ -15,6 +15,8 @@ dist_user=cvttdist
 dist_host="cloud21.cvtt.vpn"
 dist_ssh_port="22"
 
+dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+
 interactive=Y
 
 # cmdline
@@ -119,11 +121,16 @@ cmd_arr+=("${Cmd}")
 
 dist_path="${dist_root}/${project}/${release_version}"
 
-Cmd="rsync -avzh"
+for dist_loc in ${dist_locations}; do
-Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path} && rsync\""
+    dhp=(${dist_loc//:/ })
-Cmd="${Cmd} -e \"ssh -p ${dist_ssh_port}\""
+    dist_host=${dhp[0]}
-Cmd="${Cmd} ../${project} ${dist_user}@${dist_host}:${dist_path}/"
+    dist_port=${dhp[1]}
-cmd_arr+=("${Cmd}")
+    Cmd="rsync -avzh"
+    Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path} && rsync\""
+    Cmd="${Cmd} -e \"ssh -p ${dist_ssh_port}\""
+    Cmd="${Cmd} ../${project} ${dist_user}@${dist_host}:${dist_path}/"
+    cmd_arr+=("${Cmd}")
+done
 
 if [ "${interactive}" == "Y" ]; then
     echo "------------------------------------"
@@ -139,7 +146,7 @@ confirm
 
 for cmd in "${cmd_arr[@]}"
 do
-    echo ${cmd} && eval ${cmd}
+    pwd && echo ${cmd} && eval ${cmd}
 done
 
 cd ${HOME}

scripts/set_version.sh

@@ -15,11 +15,11 @@ fi
 
 if [ "${Version}" == "" ]
 then
-    usage
+    Version=latest
 fi
 
 # ----- Settings
-LocalSoftwareDir=${HOME}/software
+LocalSoftwareDir=${HOME}/software/cvtt2
 
 ProdDir=${HOME}/prod
 
@@ -29,8 +29,6 @@ ReleaseUsers=("cvttdist")
 ReleaseDir=("/home/cvttdist/software/cvtt2")
 # ----- Settings
 
-Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
-
 function rsync_load_version() {
     for idx in "${!ReleaseHosts[@]}"
     do
@@ -39,6 +37,11 @@ function rsync_load_version() {
         user=${ReleaseUsers[${idx}]}
         rel_dir=${ReleaseDir[${idx}]}
 
+        if [ "${Version}" == "latest" ]; then
+            echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+            Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+            echo "Latest version is ${Version}"
+        fi
         echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
         if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
         then
@@ -67,16 +70,14 @@ function rsync_load_version() {
 mkdir -p ${LocalSoftwareDir}
 
 # exists and not empty
-if [[ ! -d "${Location}" ]] ; then
+rsync_load_version
-    rsync_load_version
+
-fi
+Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
 
 Cmd="cd ${ProdDir}"
-
 Cmd="${Cmd} && rm -rf ${Project}"
 Cmd="${Cmd} && ln -snf ${Location} ${Project}"
 
-echo ${Cmd}
+echo ${Cmd} && eval ${Cmd}
-eval ${Cmd}
 
 echo "Done: $0 $*"

utils/set_ops_version.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-
-function usage() {
-    echo "Usage: ${0} <version>"
-    exit 1
-
-}
-Version=${1}
-
-if [ "${Version}" == "" ]
-then
-    usage
-fi
-
-Project=ops
-# ----- Settings
-LocalSoftwareDir=/opt/software/cvtt2
-
-ProdDir=${HOME}/prod
-
-Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
-
-
-ReleaseHosts=("cryptovaltrading.com" "cryptoval1.sheynin.home" "cloudstore.cvtt.vpn")
-ReleasePorts=("1588" "22" "22")
-ReleaseUsers=("cryptova" "cvtt" "cvtt")
-ReleaseDir=("/home/cryptova/opt/software/cvtt2" "/opt/software/cvtt2" "/opt/software/cvtt2")
-
-# ----- Settings
-
-function rsync_load_version() {
-    for idx in "${!ReleaseHosts[@]}"
-    do
-        host=${ReleaseHosts[${idx}]}
-        port=${ReleasePorts[${idx}]}
-        user=${ReleaseUsers[${idx}]}
-        rel_dir=${ReleaseDir[${idx}]}
-
-        echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
-        if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
-        then
-            echo "Directory found..."
-            rsync_cmd="rsync -ahvv -e \"ssh -p ${port}\""
-            rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}"
-            rsync_cmd="${rsync_cmd} ${LocalSoftwareDir}/${Project}/"
-            echo ${rsync_cmd}
-            eval ${rsync_cmd}
-            status=$?
-            if [ ${status} -eq 0 ]
-            then
-                echo "Loading successful..."
-                break
-            fi
-        else
-            echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
-        fi
-    done
-    if [[ ! -d ${LocalSoftwareDir}/${Project} ]] ; then
-        echo ERROR loading software
-        exit 1
-    fi
-}
-
-
-# exists and not empty
-if [[ ! -d "${Location}" ]] ; then
-    rsync_load_version
-fi
-
-Cmd="cd ${ProdDir}"
-
-Cmd="${Cmd} && rm -rf ${Project}"
-Cmd="${Cmd} && ln -snf ${Location} ${Project}"
-
-echo ${Cmd}
-eval ${Cmd}
-
-echo "Done: $0 $*"