progress

configs/cloud11/supervisord.conf

@@ -1,47 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[program:bnbfut_rec_cv1]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBFUT
-        --instruments=PERP-BTC-USDT,PERP-ETH-USDT
-        --db_credentials_key=TSDB_MD_CV1
-        --admin_port=7226
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CV1.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:coinbase_rec_cv1]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=COINBASE_AT
-        --instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-        --db_credentials_key=TSDB_MD_CV1
-        --admin_port=7224
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CV1.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-

configs/cloud16/supervisor.conf

@@ -1,28 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[program:config_service]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
-        -u /home/cvtt/prod/cvttpy/apps/utils/config_server.py
-        --port=6789
-        --root=/home/cvtt/prod/config_service/data
-        --log_file=/home/cvtt/prod/logs/%%T.CONFIG_SERVICE.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-

configs/cloud17/supervisord.conf

@@ -1,99 +0,0 @@
-[supervisord]
-environment=PYTHONPATH="/home/cvtt/prod"
-autostart=true
-autorestart=unexpected
-startretries=3
-logfile=/home/cvtt/prod/logs/supervisord.log
-
-[supervisorctl]
-
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-# [program:executor]
-# directory=/home/cvtt/prod
-# command=/home/cvtt/.pyenv/python3.10-venv/bin/python3
-#         -u /home/cvtt/prod/cvttpy/apps/executor_app.py
-#         --config=http://cloud16.cvtt.vpn:6789/apps/cvtt
-#         --active_exchanges=COINBASE_AT
-#         --admin_port=7220
-#         --log_level=INFO
-#         --log_file=/home/cvtt/prod/logs/%%T.EXECUTOR_APP.log
-# autostart=true
-# autorestart=true
-# startretries=1
-# startsecs=3
-# user=cvtt
-
-#[group:bnbspot_mdrec]
-#programs=
-#    bnbspot_rec_cloud17,
-#    bnbspot_rec_cv1
-
-[program:bnbfut_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBFUT
-        --instruments=PERP-BTC-USDT,PERP-ETH-USDT
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7201
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:coinbase_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=COINBASE_AT
-        --instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7202
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-[program:bnbspot_rec_cld19]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBSPOT
-        --instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-        --db_credentials_key=TSDB_MD_CLD19
-        --admin_port=7203
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CLD19.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt
-
-
-[program:bnbspot_rec_cvttdata]
-directory=/home/cvtt/prod
-command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
-        --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
-        --active_exchanges=BNBSPOT
-        --instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
-        --db_credentials_key=TSDB_MD_CVTTDATA
-        --admin_port=7222
-        --log_level=INFO
-        --log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CVTTDATA.log
-autostart=true
-autorestart=true
-startretries=1
-startsecs=3
-user=cvtt

doc/crypto_md.txt

@@ -0,0 +1,42 @@
+-------------------------------------
+C R Y P T O  M A R K E T   D A T A
+-------------------------------------
+============
+ Exchanges
+============
+	|       Coinbase
+	|       Binance Spot
+	|       Binance Futures
+	|
+ 	v
+============
+ Databases
+============
+	|       TimescaleDB: cloud21
+	|       TimescaleDB: cvttdata
+	v
+================
+ Daily Archive
+================
+    |       created by crontabs:
+    |       cvttdata:
+    |         (1) /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21
+    |         (2) /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cloud21 (1)
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cvttdata (2)
+    |
+    v
+==========
+ Storage
+==========
+    |       created by crontab cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh (ops/utls)
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
+    |
+    v
+==========
+ Usage
+==========

doc/equity_md.txt

@@ -0,0 +1,33 @@
+-------------------------------------
+E Q U I T Y  M A R K E T   D A T A
+-------------------------------------
+=====================
+ Exchanges (Sources)
+=====================
+	|       Alpaca
+	|
+    |
+    |                     TBD  ============
+    |                     TBD  Databases
+    |                     TBD  ============
+    |                     TBD      |       TimescaleDB: cloud21
+    |                     TBD      |       TimescaleDB: cvttdata
+    v                     TBD      v
+================
+ Daily Archive
+================
+    |       created by crontab on cvttdata:
+    |         /home/cvtt/prod/run/alpaca_md.sh
+    |       stored in:
+    |         cvttdata:/home/cvtt/prod/archive/md_archive/equity/alpaca_md 
+    |
+    v
+==========
+ Storage
+==========
+    |       created by crontab cvttdata:
+    |           /home/cvtt/prod/utils/sync_market_data.sh
+    |       stored in:
+    |         homestore:/works/cvtt/md_archive/equity/alpaca_md
+    |         cloud21:/opt/store/cvtt/md_archive/equity/alpaca_md
+____V_______

release_version.txt

@@ -1 +1 @@
-0.5.5
+1.0.3

scripts/ad-hoc/rearrange_alpaca_md.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}
+
+SourceRoot=/works/cvtt/md_archive/equity/alpaca_md.2
+TargetRoot=/works/cvtt/md_archive/equity/alpaca_md.NEW
+
+mkdir -p ${TargetDir}
+
+unalias ls
+cd ${SourceRoot}
+
+for year in $(ls -d 2*)
+do
+    echo $year
+    pushd $year
+    for letter in $(ls -d ?)
+    do
+        pushd ${letter}
+
+        echo "${year}/${letter}"
+        for symbol in $(ls -d ${letter}*)
+        do
+           pushd $symbol
+
+           echo "${year}/${letter}/${symbol}/* --> ${letter}/${symbol}/${year}/"
+           mkdir -p ${TargetRoot}/${letter}/${symbol}/${year}
+           mv ${SourceRoot}/${year}/${letter}/${symbol}/* ${TargetRoot}/${letter}/${symbol}/${year}/
+
+           popd
+        done
+
+        popd
+    done
+    popd
+done

scripts/alpaca_hist_md/alpaca_assets.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+CredsFile=${1}
+CredKey=${2}
+
+if [ "${CredsFile}" == "" ] ; then
+    CredsFile="${HOME}/.creds"
+fi
+
+if [ "${CredKey}" == "" ] ; then
+    CredKey="ALPACA_SANDBOX"
+fi
+
+key=$(jq -r ".[\"${CredKey}\"] | .api_key" ${CredsFile})
+secret=$(jq -r ".[\"${CredKey}\"] | .secret_key" ${CredsFile})
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+
+eval ${Cmd}
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done

scripts/alpaca_hist_md/alpaca_md.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+Start=${1}
+NumJobs=${2}
+InstListFile=${3}
+
+export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+if [ "${Start}" == "" ] ; then
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+fi
+
+if ! is_business_day ${Start}; then
+    echo "${Start} is not business day in US"
+    exit
+fi
+
+End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
+
+# Exclude today
+if [ "${End}" == $(date '+%Y-%m-%d') ] ; then
+    End=$(date -d 'yesterday' ''+%Y-%m-%d'')
+fi
+
+if [ "${NumJobs}" == "" ] ; then
+    NumJobs=30
+fi
+
+echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
+
+# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
+
+export PYTHONPATH=/home/cvtt/prod
+export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+
+export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local
+export LogDir=/home/cvtt/prod/logs/alpaca_md
+
+mkdir -p ${LogDir}
+
+run_proc() {
+    Inst=${1}
+    Start=${2}
+    End=${3}
+    echo "Running for $Inst"
+
+    Cmd="${Python} ${PyScript}"
+    Cmd="${Cmd} --config=${Config}"
+    Cmd="${Cmd} --output_dir=${OutputDir}/${Start:0:4}"
+    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
+    Cmd="${Cmd} --start=${Start}"
+    if [ "${End}" != "" ]; then
+        Cmd="${Cmd} --end=${End}"
+    fi
+    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
+    Cmd="${Cmd} --log_level=WARNING"
+    echo ${Cmd}
+    eval ${Cmd}
+}
+
+export -f run_proc
+
+if [ "$InstListFile" == "" ]; then
+
+    key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
+    secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
+
+    Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+    Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+    Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+    Cmd="${Cmd} --header 'accept: application/json'"
+    Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
+    Cmd="${Cmd}  | sed 's/\"//g'"
+    Cmd="${Cmd} | sed 's/^/STOCK-/'"
+else
+    Cmd="cat ${InstListFile}"
+fi
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
+
+slice_size=500 # 10K symbols parallel cannot handle
+for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
+    InstSlice=("${Instruments[@]:ii:slice_size}")
+
+    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
+done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done

scripts/alpaca_hist_md/alpaca_md_day.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+date_to_load=${1}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/alpaca_md_day:latest
+LastDayFile=/home/cvtt/prod/logs/last_alpaca_hist_day
+
+if [ -z ${date_to_load} ] ; then
+    echo "Yesterday data. Historical container will be killed"
+    docker kill alpaca_md_day
+else
+    if [ "${date_to_load}" == "next" ] ; then
+        last_date=$(cat ${LastDayFile})
+        if [ -z ${last_date} ] ; then
+            echo "No last date_to_load in ${LastDayFile}"
+            exit 1
+        fi
+        date_to_load=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    fi
+    echo "Historical Data for ${date_to_load}"
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=alpaca_md_day"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} ${date_to_load}"
+
+echo $Cmd
+eval $Cmd
+
+if [ "$?" != "0" ] ; then
+    exit 1 # if killed we do not save last day
+fi
+
+if [ -n ${date_to_load}]; then
+    echo "Saving date_to_load to ${LastDayFile}"
+    echo ${date_to_load} >  ${LastDayFile}
+fi

scripts/alpaca_hist_md/run_alpaca_hist_md.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+

scripts/archive_yesterday_md.sh

@@ -4,14 +4,14 @@ Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
 RootDir=/home/cvtt/prod
 export PYTHONPATH=${RootDir}
 
-host=$(hostname)
+host=${1}
 if [ "${host}" == "cvttdata" ]
 then
-    ArchiveRootDir=/home/cvtt/prod/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cvttdata
     CredKey=TSDB_MD_CVTTDATA_RO
-elif [ "${host}" == "cloud21.cryptovaltrading.com" ]
+elif [ "${host}" == "cloud21" ]
 then
-    ArchiveRootDir=/opt/store/cvtt/archive/md_archive
+    ArchiveRootDir=/home/cvtt/prod/archive/md_archive/crypto/cloud21
     CredKey=TSDB_MD_CLD21_RO
 else
     echo "Unknown host ${host}. ${0} Aborted."
@@ -21,7 +21,7 @@ fi
 mkdir -p ${ArchiveRootDir}
 
 yesterday=$(date -d "yesterday" +%Y%m%d)
-Schemas=${1}
+Schemas=${2}
 if [ "${Schemas}" == "" ]
 then
     Schemas="coinbase,bnbspot,bnbfut"

scripts/build_release.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+# ---------------- Settings
+declare -A git_repo_arr
+
+git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
+git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
+git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
+
+default_project=cvttpy
+default_branch=master
+
+
+dist_root=/home/cvttdist/software/cvtt2
+dist_user=cvttdist
+dist_host="cloud21.cvtt.vpn"
+dist_ssh_port="22"
+
+dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+
+interactive=Y
+
+# cmdline
+prj=${1:-${default_project}}
+brnch=${2:-${default_branch}}
+if [ "${3}" == "D" ]; then
+    interactive=N
+fi
+
+function confirm {
+    if [ "${interactive}" == "Y" ]; then
+        echo "--------------------------------"
+        echo -n "Press <Enter> to continue" && read
+    fi
+}
+
+if [ "${interactive}" == "Y" ]; then
+    echo -n "Enter project [${prj}]: "
+    read project
+    if [ "${project}" == "" ]
+    then
+        project=${prj}
+    fi
+else
+    project=${prj}
+fi
+
+repo=${git_repo_arr[${project}]}
+if [ -z ${repo} ]; then
+    echo "ERROR: Project repository for ${project} not found"
+    exit -1
+fi
+echo "Project repo: ${repo}"
+
+if [ "${interactive}" == "Y" ]; then
+    echo -n "Enter branch to build release from [${brnch}]: "
+    read branch
+    if [ "${branch}" == "" ]
+    then
+        branch=${brnch}
+    fi
+else
+    branch=${brnch}
+fi
+
+tmp_dir=$(mktemp -d)
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+
+prj_dir="${tmp_dir}/${prj}"
+
+cmd_arr=()
+Cmd="git clone ${repo} ${prj_dir}"
+cmd_arr+=("${Cmd}")
+
+Cmd="cd ${prj_dir}"
+cmd_arr+=("${Cmd}")
+
+if [ "${interactive}" == "Y" ]; then
+    echo "------------------------------------"
+    echo "The following commands will execute:"
+    echo "------------------------------------"
+    for cmd in "${cmd_arr[@]}"
+    do
+        echo ${cmd}
+    done
+fi
+
+confirm
+
+for cmd in "${cmd_arr[@]}"
+do
+    echo ${cmd} && eval ${cmd}
+done
+
+Cmd="git checkout ${branch}"
+echo ${Cmd} && eval ${Cmd}
+if [ "${?}" != "0" ]; then
+    echo "ERROR: Branch ${branch} is not found"
+    cd ${HOME} && rm -rf ${tmp_dir}
+    exit -1
+fi
+
+
+release_version=$(cat release_version.txt)
+
+
+echo "--------------------------------"
+echo "Release version: ${release_version}"
+
+confirm
+
+version_tag="v${release_version}"
+version_comment="${version_tag}_${project}_${branch}_$(date +%Y%m%d)"
+
+cmd_arr=()
+Cmd="git tag -a ${version_tag} -m ${version_comment}"
+cmd_arr+=("${Cmd}")
+
+Cmd="git push origin --tags"
+cmd_arr+=("${Cmd}")
+
+Cmd="rm -rf .git"
+cmd_arr+=("${Cmd}")
+
+dist_path="${dist_root}/${project}/${release_version}"
+
+for dist_loc in ${dist_locations}; do
+    dhp=(${dist_loc//:/ })
+    dist_host=${dhp[0]}
+    dist_port=${dhp[1]}
+    Cmd="rsync -avzh"
+    Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path} && rsync\""
+    Cmd="${Cmd} -e \"ssh -p ${dist_ssh_port}\""
+    Cmd="${Cmd} ../${project} ${dist_user}@${dist_host}:${dist_path}/"
+    cmd_arr+=("${Cmd}")
+done
+
+if [ "${interactive}" == "Y" ]; then
+    echo "------------------------------------"
+    echo "The following commands will execute:"
+    echo "------------------------------------"
+    for cmd in "${cmd_arr[@]}"
+    do
+        echo ${cmd}
+    done
+fi
+
+confirm
+
+for cmd in "${cmd_arr[@]}"
+do
+    pwd && echo ${cmd} && eval ${cmd}
+done
+
+echo "$0 Done ${project} ${release_version}"

scripts/crypto_hist_md/crypto_md_day.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+host=${1}
+date=${2}
+
+DockerRegistry=cloud21.cvtt.vpn:5500
+DockerImage=${DockerRegistry}/crypto_md_day
+
+if [ -z ${date} ] ; then
+    date=$(date -d "yesterday" +%Y%m%d)
+fi
+
+Cmd="docker run"
+Cmd="${Cmd} --network=host"
+Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
+Cmd="${Cmd} --rm"
+Cmd="${Cmd} ${DockerImage}"
+Cmd="${Cmd} -h ${host}"
+Cmd="${Cmd} -d ${date}"
+Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
+
+echo $Cmd
+eval $Cmd

scripts/crypto_hist_md/process_daily_md.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+date=${1}
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+SourceHost=cryptoval3.cvtt.vpn
+SourceUser=cvtt
+SourceRootDir=/localdisk/cvtt/archive/md_archive
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then    
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file

scripts/healthcheck/storage_health_check.sh

@@ -15,7 +15,6 @@ Sender=${RootDir}/ops/utils/send_mmost.sh
 
 Hosts=
 Hosts="${Hosts} cloud11.cvtt.vpn"
-Hosts="${Hosts} cloud14.cvtt.vpn"
 Hosts="${Hosts} cloud15.cvtt.vpn"
 Hosts="${Hosts} cloud16.cvtt.vpn"
 Hosts="${Hosts} cloud17.cvtt.vpn"
@@ -23,23 +22,38 @@ Hosts="${Hosts} cloud21.cvtt.vpn"
 Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
 Hosts="${Hosts} cvttdata.cvtt.vpn"
-Hosts="${Hosts} cryptoval1.cvtt.vpn"
 Hosts="${Hosts} cryptoval2.cvtt.vpn"
 Hosts="${Hosts} cryptoval3.cvtt.vpn"
 
 Hosts="${Hosts} homestore.cvtt.vpn"
-Hosts="${Hosts} nsbackup.cvtt.vpn"
+Hosts="${Hosts} nsbackup.sheynin.home"
 Hosts="${Hosts} dtvmhost.cvtt.vpn"
 Hosts="${Hosts} ops-server.cvtt.vpn"
 
+Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
+Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
+Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
+
+Metrics=()
+TempFiles=
+
+function cleanup {
+    echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
+
 function space_alert() {
     ALERT_USAGE=75%
-    for ln in "${Measurements[@]}"
+    for metric in "${Metrics[@]}"
     do
-        IFS=$' '; args=($ln); unset IFS
+        IFS=$' '; args=($metric); unset IFS
         host=${args[0]}
         fs=${args[1]}
         space_used=${args[2]}
+
         if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
             echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
         fi
@@ -50,6 +64,7 @@ function storage_check() {
 
     result_lines=()
 
+    declare -a SingleMeas
     for host in ${Hosts}
     do
         if [[ "${host}" == *"cryptovaltrading.com" ]]; then
@@ -68,7 +83,12 @@ function storage_check() {
         do
             IFS=$' '; args=($ln); unset IFS
             res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
+
             result_lines+=("$res")
+
+            fs=${args[6]}
+            used=${args[5]}
+            Metrics+=("$host $fs $used")
         done
         unset IFS
     done
@@ -79,7 +99,11 @@ function storage_check() {
 }
 
 tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 tmpfile2=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile2}"
+
 storage_check > ${tmpfile2}
 
 echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
@@ -88,17 +112,15 @@ echo "| host | filesystem | usage |"  >> ${tmpfile}
 echo "| --- | --- | --- |"  >> ${tmpfile}
 cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
 cat ${tmpfile} |  ${Sender} ${StatusChannel}
-rm ${tmpfile} ${tmpfile2}
 
-Measurements=()
 tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
 space_alert > ${tmpfile}
-cat ${tmpfile}
 if [ -s ${tmpfile} ]
 then
     (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
 else
-    echo File ${tmpfile} is empty
+    echo "No Storage Alerts"
 fi
-rm ${tmpfile}
 

scripts/legacy_eqt_md/process_md.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Collect all "A" files
+letter=${1}
+
+if [ "${letter}" == "" ]; then
+	echo "Usage $0 <letter>"
+	exit 1
+fi
+
+# copy text data to local location
+rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
+
+# create db files
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+
+if [ "$?" != "0" ] ; then
+    exit
+fi
+
+# Move all files to host drive
+# a) create file list
+(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files
+
+echo ${letter} is done
+exit
+
+
+# b) rsync files to host drive
+rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/
+
+# Clean directories
+( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}

scripts/set_version.sh

@@ -1,32 +1,34 @@
 #!/bin/bash
 
+
+# ----- Settings
+LocalSoftwareDir=${HOME}/software/cvtt2
+
+ProdDir=${HOME}/prod
+
+ReleaseHosts=("cloud21.cvtt.vpn")
+ReleasePorts=("22")
+ReleaseUsers=("cvttdist")
+ReleaseDir=("/home/cvttdist/software/cvtt2")
+# ----- Settings
+
 function usage() {
-    echo "Usage: ${0} <version>"
+    echo "Usage: ${0} <project> <version>"
     exit 1
 
 }
-Version=${1}
+Project=${1}
+Version=${2}
 
-if [ "${Version}" == "" ]
+if [ "${Project}" == "" ]
 then
     usage
 fi
 
-Project=ops
+if [ "${Version}" == "" ]
-# ----- Settings
+then
-LocalSoftwareDir=/opt/software/cvtt2
+    Version=latest
-
+fi
-ProdDir=${HOME}/prod
-
-Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
-
-
-ReleaseHosts=("cryptovaltrading.com" "cryptoval1.sheynin.home" "cloudstore.cvtt.vpn")
-ReleasePorts=("1588" "22" "22")
-ReleaseUsers=("cryptova" "cvtt" "cvtt")
-ReleaseDir=("/home/cryptova/opt/software/cvtt2" "/opt/software/cvtt2" "/opt/software/cvtt2")
-
-# ----- Settings
 
 function rsync_load_version() {
     for idx in "${!ReleaseHosts[@]}"
@@ -36,6 +38,11 @@ function rsync_load_version() {
         user=${ReleaseUsers[${idx}]}
         rel_dir=${ReleaseDir[${idx}]}
 
+        if [ "${Version}" == "latest" ]; then
+            echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+            Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+            echo "Latest version is ${Version}"
+        fi
         echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
         if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
         then
@@ -61,18 +68,18 @@ function rsync_load_version() {
     fi
 }
 
+mkdir -p ${LocalSoftwareDir}
+mkdir -p ${ProdDir}
 
 # exists and not empty
-if [[ ! -d "${Location}" ]] ; then
+rsync_load_version
-    rsync_load_version
+
-fi
+Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
 
 Cmd="cd ${ProdDir}"
-
 Cmd="${Cmd} && rm -rf ${Project}"
 Cmd="${Cmd} && ln -snf ${Location} ${Project}"
 
-echo ${Cmd}
+echo ${Cmd} && eval ${Cmd}
-eval ${Cmd}
 
 echo "Done: $0 $*"

utils/archive_logs.sh

@@ -26,7 +26,7 @@ Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d')
 
 echo "Looking for log files older than ${DateCriteria} in ${LogDir}"
 
-LogArchiveDir=/home/cvtt/prod/archive/logs
+LogArchiveDir=${HOME}/prod/archive/logs
 mkdir -p ${LogArchiveDir}
 
 echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')'  -type f -not -newermt ${Oldest})"

utils/prune_data.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+Src=${1}
+Days=${2}
+
+if [ -z "$Days" ] || [ -z "$Src" ]; then
+    echo "Usage: $0 <source_dir> <num_days>"
+    exit 1
+fi
+
+# Add / if Src does not have it (symlinks)
+if [ "${Src: -1}" != "/" ]; then
+    Src="${Src}/"
+fi
+
+declare -A Settings=()
+Settings[Src]=${Src}
+Settings[PruneDate]=$(date -d "${Days} days ago" '+%Y-%m-%d')
+
+src=${Settings[Src]}
+prune_date=${Settings[PruneDate]}
+
+
+echo "Finding files older than ${prune_date} in ${Src} ..."
+Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
+echo ${Cmd}
+files=($(eval ${Cmd}))
+total_files=${#files[*]}
+
+if [[ ${total_files} == 0 ]]
+then
+    echo "No files found to be pruned. Bye..."
+    exit 0
+fi
+
+echo Before Pruning....
+duf ${src}
+
+echo "The following files will be removed:"
+echo "===================================="
+for f in $files ; do  ls -l $f; done
+echo "===================================="
+echo "Total files to be pruned: ${total_files}"
+
+echo "Removing files..."
+Cmd="${Cmd} -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Removing empty directories..."
+Cmd="find ${src} -type d -empty -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo After Pruning....
+duf ${src}
+
+echo $0 Done

utils/send_mmost.sh

@@ -11,6 +11,10 @@ url=$(jq -r '.MATTERMOST.url' ${HOME}/.creds)
 team=$(jq -r '.MATTERMOST.team' ${HOME}/.creds)
 bearer=$(jq -r '.MATTERMOST.bearer' ${HOME}/.creds)
 
+#lowercase channel
+channel_name=$(echo "${channel_name}" | awk '{print tolower($0)}')
+# space to '-'
+channel_name=${channel_name//" "/"-"}
 
 get_channel_url="${url}/api/v4/teams/name/${team}/channels/name/${channel_name}"
 

utils/sync_market_data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Source=/home/cvtt/prod/archive/md_archive/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done