diff --git a/scripts/archive_yesterday_md.sh b/scripts/__DEPRECATED__/archive_yesterday_md.sh similarity index 100% rename from scripts/archive_yesterday_md.sh rename to scripts/__DEPRECATED__/archive_yesterday_md.sh diff --git a/scripts/alpaca_hist_md/alpaca_assets.sh b/scripts/mkt_data/alpaca_hist_md/alpaca_assets.sh similarity index 100% rename from scripts/alpaca_hist_md/alpaca_assets.sh rename to scripts/mkt_data/alpaca_hist_md/alpaca_assets.sh diff --git a/scripts/alpaca_hist_md/alpaca_md.sh b/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh similarity index 100% rename from scripts/alpaca_hist_md/alpaca_md.sh rename to scripts/mkt_data/alpaca_hist_md/alpaca_md.sh diff --git a/scripts/alpaca_hist_md/load_rel_liq.sh b/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh similarity index 100% rename from scripts/alpaca_hist_md/load_rel_liq.sh rename to scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh diff --git a/scripts/alpaca_hist_md/run_alpaca_hist_md.sh b/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh similarity index 100% rename from scripts/alpaca_hist_md/run_alpaca_hist_md.sh rename to scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh diff --git a/scripts/mkt_data/checklists/crypto_daily_chcklst.sh b/scripts/mkt_data/checklists/crypto_daily_chcklst.sh new file mode 100755 index 0000000..6754632 --- /dev/null +++ b/scripts/mkt_data/checklists/crypto_daily_chcklst.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +usage() { + echo -n "Usage: ${0}" + echo + exit 1 +} + + +RootDir="${HOME}/prod" +RootDir=/home/oleg/develop/cvtt2 ###### D E B U G + +export PYTHONPATH=${RootDir} + +StatusChannel="MD-Status" +Sender=${RootDir}/ops/utils/send_mmost.sh + +# ----- For DEBUGGING +# Sender=cat +# StatusChannel= + +run_checklist() { + yr=$(date -d 'yesterday' '+%Y') + mn=$(date -d 'yesterday' '+%m') + + declare -A Commands + Commands=( + ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'" + ) + + for name in $(printf "%s\n" "${!Commands[@]}" | sort); do + Cmd=${Commands[${name}]} + echo "------- ${name}" + echo ${Cmd} + eval ${Cmd} + done +} + +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + +echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile} +echo '```' >> ${tmpfile} +run_checklist >> ${tmpfile} +echo '```' >> ${tmpfile} + +cat ${tmpfile} | ${Sender} ${StatusChannel} + +cat $tmpfile + diff --git a/scripts/mkt_data/checklists/eqty_md_checklist.sh b/scripts/mkt_data/checklists/eqty_md_checklist.sh new file mode 100755 index 0000000..b240309 --- /dev/null +++ b/scripts/mkt_data/checklists/eqty_md_checklist.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +usage() { + echo -n "Usage: ${0}" + echo + exit 1 +} + +RootDir="${HOME}/prod" +# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G + +export PYTHONPATH=${RootDir} + +StatusChannel="MD-Status" +Sender=${RootDir}/ops/utils/send_mmost.sh + +# ----- For DEBUGGING +# Sender=cat +# StatusChannel= + +run_checklist() { + + + yr=$(date -d 'yesterday' '+%Y') + + declare -A Commands + Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3" + Commands["homestore"]+="; echo" + Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3" + + Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3" + Commands["cloud21"]+="; echo" + Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3" + + Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'" + + for name in $(printf "%s\n" "${!Commands[@]}" | sort); do + Cmd=${Commands[${name}]} + echo "------- ${name}" + echo ${Cmd} + eval ${Cmd} + done +} + +# run_checklist + +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + +echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile} +echo '```' >> ${tmpfile} +run_checklist >> ${tmpfile} +echo '```' >> ${tmpfile} + +cat ${tmpfile} | ${Sender} ${StatusChannel} + + + +exit +# homestore +echo homestore +ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3 +echo +ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3 + +echo cloud21 +ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3 +echo +ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3 + +echo "------ gpushnik" +ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10' \ No newline at end of file diff --git a/scripts/crypto_hist_md/exchanges_trading_stats.sh b/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh similarity index 100% rename from scripts/crypto_hist_md/exchanges_trading_stats.sh rename to scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh diff --git a/scripts/crypto_hist_md/process_daily_md.sh b/scripts/mkt_data/crypto_hist_md/process_daily_md.sh similarity index 100% rename from scripts/crypto_hist_md/process_daily_md.sh rename to scripts/mkt_data/crypto_hist_md/process_daily_md.sh diff --git a/scripts/legacy_eqt_md/process_md.sh b/scripts/mkt_data/legacy_eqt_md/process_md.sh similarity index 100% rename from scripts/legacy_eqt_md/process_md.sh rename to scripts/mkt_data/legacy_eqt_md/process_md.sh diff --git a/utils/sync_market_data.sh b/scripts/mkt_data/sync_market_data.sh similarity index 100% rename from utils/sync_market_data.sh rename to scripts/mkt_data/sync_market_data.sh