From b57fb2257fe3023758271a8962508ba5b6feae2f Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Sun, 18 Aug 2024 16:29:40 -0400 Subject: [PATCH] progress - intermediate for md checklists --- .../archive_yesterday_md.sh | 0 .../alpaca_hist_md/alpaca_assets.sh | 0 .../alpaca_hist_md/alpaca_md.sh | 0 .../alpaca_hist_md/load_rel_liq.sh | 0 .../alpaca_hist_md/run_alpaca_hist_md.sh | 0 .../checklists/crypto_daily_chcklst.sh | 58 ++++++++++++++ .../mkt_data/checklists/eqty_md_checklist.sh | 76 +++++++++++++++++++ .../crypto_hist_md/exchanges_trading_stats.sh | 0 .../crypto_hist_md/process_daily_md.sh | 0 .../legacy_eqt_md/process_md.sh | 0 .../mkt_data}/sync_market_data.sh | 0 11 files changed, 134 insertions(+) rename scripts/{ => __DEPRECATED__}/archive_yesterday_md.sh (100%) rename scripts/{ => mkt_data}/alpaca_hist_md/alpaca_assets.sh (100%) rename scripts/{ => mkt_data}/alpaca_hist_md/alpaca_md.sh (100%) rename scripts/{ => mkt_data}/alpaca_hist_md/load_rel_liq.sh (100%) rename scripts/{ => mkt_data}/alpaca_hist_md/run_alpaca_hist_md.sh (100%) create mode 100755 scripts/mkt_data/checklists/crypto_daily_chcklst.sh create mode 100755 scripts/mkt_data/checklists/eqty_md_checklist.sh rename scripts/{ => mkt_data}/crypto_hist_md/exchanges_trading_stats.sh (100%) rename scripts/{ => mkt_data}/crypto_hist_md/process_daily_md.sh (100%) rename scripts/{ => mkt_data}/legacy_eqt_md/process_md.sh (100%) rename {utils => scripts/mkt_data}/sync_market_data.sh (100%) diff --git a/scripts/archive_yesterday_md.sh b/scripts/__DEPRECATED__/archive_yesterday_md.sh similarity index 100% rename from scripts/archive_yesterday_md.sh rename to scripts/__DEPRECATED__/archive_yesterday_md.sh diff --git a/scripts/alpaca_hist_md/alpaca_assets.sh b/scripts/mkt_data/alpaca_hist_md/alpaca_assets.sh similarity index 100% rename from scripts/alpaca_hist_md/alpaca_assets.sh rename to scripts/mkt_data/alpaca_hist_md/alpaca_assets.sh diff --git a/scripts/alpaca_hist_md/alpaca_md.sh b/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh similarity index 100% rename from scripts/alpaca_hist_md/alpaca_md.sh rename to scripts/mkt_data/alpaca_hist_md/alpaca_md.sh diff --git a/scripts/alpaca_hist_md/load_rel_liq.sh b/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh similarity index 100% rename from scripts/alpaca_hist_md/load_rel_liq.sh rename to scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh diff --git a/scripts/alpaca_hist_md/run_alpaca_hist_md.sh b/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh similarity index 100% rename from scripts/alpaca_hist_md/run_alpaca_hist_md.sh rename to scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh diff --git a/scripts/mkt_data/checklists/crypto_daily_chcklst.sh b/scripts/mkt_data/checklists/crypto_daily_chcklst.sh new file mode 100755 index 0000000..6754632 --- /dev/null +++ b/scripts/mkt_data/checklists/crypto_daily_chcklst.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +usage() { + echo -n "Usage: ${0}" + echo + exit 1 +} + + +RootDir="${HOME}/prod" +RootDir=/home/oleg/develop/cvtt2 ###### D E B U G + +export PYTHONPATH=${RootDir} + +StatusChannel="MD-Status" +Sender=${RootDir}/ops/utils/send_mmost.sh + +# ----- For DEBUGGING +# Sender=cat +# StatusChannel= + +run_checklist() { + yr=$(date -d 'yesterday' '+%Y') + mn=$(date -d 'yesterday' '+%m') + + declare -A Commands + Commands=( + ["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'" + ) + + for name in $(printf "%s\n" "${!Commands[@]}" | sort); do + Cmd=${Commands[${name}]} + echo "------- ${name}" + echo ${Cmd} + eval ${Cmd} + done +} + +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + +echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile} +echo '```' >> ${tmpfile} +run_checklist >> ${tmpfile} +echo '```' >> ${tmpfile} + +cat ${tmpfile} | ${Sender} ${StatusChannel} + +cat $tmpfile + diff --git a/scripts/mkt_data/checklists/eqty_md_checklist.sh b/scripts/mkt_data/checklists/eqty_md_checklist.sh new file mode 100755 index 0000000..b240309 --- /dev/null +++ b/scripts/mkt_data/checklists/eqty_md_checklist.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +usage() { + echo -n "Usage: ${0}" + echo + exit 1 +} + +RootDir="${HOME}/prod" +# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G + +export PYTHONPATH=${RootDir} + +StatusChannel="MD-Status" +Sender=${RootDir}/ops/utils/send_mmost.sh + +# ----- For DEBUGGING +# Sender=cat +# StatusChannel= + +run_checklist() { + + + yr=$(date -d 'yesterday' '+%Y') + + declare -A Commands + Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3" + Commands["homestore"]+="; echo" + Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3" + + Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3" + Commands["cloud21"]+="; echo" + Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3" + + Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'" + + for name in $(printf "%s\n" "${!Commands[@]}" | sort); do + Cmd=${Commands[${name}]} + echo "------- ${name}" + echo ${Cmd} + eval ${Cmd} + done +} + +# run_checklist + +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + +echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile} +echo '```' >> ${tmpfile} +run_checklist >> ${tmpfile} +echo '```' >> ${tmpfile} + +cat ${tmpfile} | ${Sender} ${StatusChannel} + + + +exit +# homestore +echo homestore +ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3 +echo +ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3 + +echo cloud21 +ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3 +echo +ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3 + +echo "------ gpushnik" +ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10' \ No newline at end of file diff --git a/scripts/crypto_hist_md/exchanges_trading_stats.sh b/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh similarity index 100% rename from scripts/crypto_hist_md/exchanges_trading_stats.sh rename to scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh diff --git a/scripts/crypto_hist_md/process_daily_md.sh b/scripts/mkt_data/crypto_hist_md/process_daily_md.sh similarity index 100% rename from scripts/crypto_hist_md/process_daily_md.sh rename to scripts/mkt_data/crypto_hist_md/process_daily_md.sh diff --git a/scripts/legacy_eqt_md/process_md.sh b/scripts/mkt_data/legacy_eqt_md/process_md.sh similarity index 100% rename from scripts/legacy_eqt_md/process_md.sh rename to scripts/mkt_data/legacy_eqt_md/process_md.sh diff --git a/utils/sync_market_data.sh b/scripts/mkt_data/sync_market_data.sh similarity index 100% rename from utils/sync_market_data.sh rename to scripts/mkt_data/sync_market_data.sh