progress - intermediate for md checklists

2024-08-18 16:29:40 -04:00
parent 1dafab8971
commit b57fb2257f
11 changed files with 134 additions and 0 deletions
@@ -0,0 +1,28 @@
+#!/bin/bash
+CredsFile=${1}
+CredKey=${2}
+
+if [ "${CredsFile}" == "" ] ; then
+    CredsFile="${HOME}/.creds"
+fi
+
+if [ "${CredKey}" == "" ] ; then
+    CredKey="ALPACA_SANDBOX"
+fi
+
+key=$(jq -r ".[\"${CredKey}\"] | .api_key" ${CredsFile})
+secret=$(jq -r ".[\"${CredKey}\"] | .secret_key" ${CredsFile})
+
+Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+Cmd="${Cmd} --header 'accept: application/json'"
+
+eval ${Cmd}
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
@@ -0,0 +1,114 @@
+#!/bin/bash
+
+Start=${1}
+NumJobs=${2}
+InstListFile=${3}
+
+export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+
+is_business_day() {
+    dt=${1}
+
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+if [ "${Start}" == "" ] ; then
+    Start=$(date -d "yesterday" "+%Y-%m-%d")
+fi
+
+if ! is_business_day ${Start}; then
+    echo "${Start} is not business day in US"
+    exit
+fi
+
+End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
+
+# Exclude today
+if [ "${End}" == $(date '+%Y-%m-%d') ] ; then
+    End=$(date -d 'yesterday' ''+%Y-%m-%d'')
+fi
+
+if [ "${NumJobs}" == "" ] ; then
+    NumJobs=30
+fi
+
+echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
+
+# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
+
+export PYTHONPATH=/home/cvtt/prod
+export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
+export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
+export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
+
+export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local
+export LogDir=/home/cvtt/prod/logs/alpaca_md
+
+mkdir -p ${LogDir}
+
+run_proc() {
+    Inst=${1}
+    Start=${2}
+    End=${3}
+    echo "Running for $Inst"
+
+    Cmd="${Python} ${PyScript}"
+    Cmd="${Cmd} --config=${Config}"
+    Cmd="${Cmd} --output_dir=${OutputDir}/${Start:0:4}"
+    Cmd="${Cmd} --instruments=ALPACA:${Inst}"
+    Cmd="${Cmd} --start=${Start}"
+    if [ "${End}" != "" ]; then
+        Cmd="${Cmd} --end=${End}"
+    fi
+    Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
+    Cmd="${Cmd} --log_level=WARNING"
+    echo ${Cmd}
+    eval ${Cmd}
+}
+
+export -f run_proc
+
+if [ "$InstListFile" == "" ]; then
+
+    key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
+    secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
+
+    Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
+    Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
+    Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
+    Cmd="${Cmd} --header 'accept: application/json'"
+    Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
+    Cmd="${Cmd}  | sed 's/\"//g'"
+    Cmd="${Cmd} | sed 's/^/STOCK-/'"
+else
+    Cmd="cat ${InstListFile}"
+fi
+
+# split string into array
+Instruments=()
+for Inst in $(eval ${Cmd})
+do
+    Instruments+=("$Inst")
+done
+
+slice_size=500 # 10K symbols parallel cannot handle
+for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
+    InstSlice=("${Instruments[@]:ii:slice_size}")
+
+    parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
+done
+
+echo "Compressing"
+for file in $(find ${OutputDir} -type f -name '*db' -print )
+do
+    echo "Compressing ${file}"
+    gzip ${file}
+done
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export PYTHONPATH=${HOME}/prod
+
+Python=${HOME}/.pyenv/python3.10-venv/bin/python3
+Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py
+DbFile=${HOME}/prod/data/rel_liquidity.db
+Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt
+
+Cmd="${Python}"
+Cmd="${Cmd} ${Script}"
+Cmd="${Cmd} --config=${Config}"
+Cmd="${Cmd} --db_file=${DbFile}"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo Done ${0} ${*}
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+is_business_day() {
+    dt=${1}
+
+    CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+    open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+# Dates are in descending order
+FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
+Script=${HOME}/prod/run/alpaca_md.sh
+
+last_date=$(cat ${FinishedDateFile})
+
+if [ "${last_date}" == "" ]; then
+    echo "process is currently on"
+    exit
+fi
+
+while true; do
+    new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
+    if is_business_day ${new_date}; then
+        break
+    else
+        last_date=${new_date}
+    fi
+done
+
+
+echo Collecting data for ${new_date}
+
+Cmd="${Script} ${new_date}"
+
+echo "${0} Starting at $(date)"
+
+echo > ${FinishedDateFile}
+echo ${Cmd} && eval ${Cmd}
+echo $new_date > ${FinishedDateFile}
+
+echo "${0} Done at $(date)"
+
+
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+
+    declare -A Commands
+    Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["homestore"]+="; echo"
+    Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+
+
+exit
+# homestore
+echo homestore
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo cloud21
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
+echo
+ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
+
+echo "------ gpushnik"
+ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PYTHONPATH=/home/cvtt/prod
+
+Cmd="/home/cvtt/.pyenv/python3.10-venv/bin/python3"
+Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py"
+Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data"
+Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN"
+Cmd="${Cmd} --instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD"
+Cmd="${Cmd} --db_file=/home/cvtt/prod/data/exchange_trading_stats.db"
+Cmd="${Cmd} --log_file=/home/cvtt/prod/logs/%%T.EXCHANGE_TRADING_STATS.log"
+
+echo ${Cmd}
+eval ${Cmd}
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+date=${1}
+if [ "${date}" == "" ] ; then
+    date="yesterday"
+fi
+
+SourceHost=cryptoval3.cvtt.vpn
+SourceUser=cvtt
+SourceRootDir=/localdisk/cvtt/archive/md_archive
+SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
+SourceFileZip="${SourceFile}.gz"
+SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
+
+TargetDir="/opt/jupyter_gpu/data/crypto_md"
+TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
+TargetFilePath="${TargetDir}/${TargetFile}"
+
+echo ${SourceFile}
+tmp_dir=$(mktemp -d)
+
+function cleanup {
+    cd ${HOME}
+    rm -rf ${tmp_dir}
+}
+trap cleanup EXIT
+
+function download_file {
+
+    Cmd="rsync"
+    Cmd="${Cmd} -ahv"
+    if tty -s; then    
+        Cmd="${Cmd} --progress=info2"
+    fi
+    Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
+    echo ${Cmd}
+    eval ${Cmd}
+    ls -lh ${tmp_dir}
+    Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
+    echo ${Cmd} && eval ${Cmd}
+    ls -lh ${tmp_dir}
+
+    rm -f ${TargetFilePath}
+    touch ${TargetFilePath}
+
+    for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
+    do
+        Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+    chmod 600 ${TargetFilePath}
+    ls -lh ${TargetFilePath}
+
+
+}
+download_file
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Collect all "A" files
+letter=${1}
+
+if [ "${letter}" == "" ]; then
+	echo "Usage $0 <letter>"
+	exit 1
+fi
+
+# copy text data to local location
+rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
+
+# create db files
+PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
+
+if [ "$?" != "0" ] ; then
+    exit
+fi
+
+# Move all files to host drive
+# a) create file list
+(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files
+
+echo ${letter} is done
+exit
+
+
+# b) rsync files to host drive
+rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/
+
+# Clean directories
+( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+Source=/home/cvtt/prod/archive/md_archive/
+Targets=
+Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+
+
+for tgt in ${Targets}
+do
+    Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
+    echo $Cmd
+    eval $Cmd
+done