From 9f789738e9263dba7ba51fbea8d6df30e61f386f Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Mon, 3 Feb 2025 15:31:34 -0500 Subject: [PATCH] using HA trading-calendar and config service --- VERSION | 2 +- build/docker_images_builder.sh | 2 +- cronjobs/save_cronjobs.sh | 2 +- docker/market_data/hist/__DEPRECATED__/alpaca_md_day.sh | 2 +- docker/market_data/hist/crypto_md_day.sh | 2 +- docker/market_data/hist/load_alpaca_hbar.sh | 4 ++-- docker/market_data/hist/load_alpaca_qat.sh | 4 ++-- docker/market_data/recorder/md_recorder.sh | 2 +- docker/market_data/recorder/md_recorder_monitor.sh | 2 +- docker/research/crypto_exch_stats.sh | 2 +- research/prepare_eqt_simdata.sh | 2 +- scripts/__DEPRECATED__/archive_yesterday_md.sh | 2 +- scripts/healthcheck/cvtt_ts_status_check.sh | 4 ++-- scripts/healthcheck/hosts_check.sh | 2 +- scripts/healthcheck/services_check.sh | 2 +- scripts/healthcheck/storage_health_check.sh | 2 +- scripts/mkt_data/alpaca_hist_md/alpaca_md.sh | 4 ++-- scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh | 2 +- scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh | 2 +- scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh | 2 +- 20 files changed, 24 insertions(+), 24 deletions(-) diff --git a/VERSION b/VERSION index 0808d67..b18be09 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.9.F2,docker build imagages script fix +2.0.1,using HA trading-calendar and config service diff --git a/build/docker_images_builder.sh b/build/docker_images_builder.sh index d9785b0..7b68548 100755 --- a/build/docker_images_builder.sh +++ b/build/docker_images_builder.sh @@ -19,7 +19,7 @@ fi RegistryService=cloud21.cvtt.vpn:5500 RegistryProtocol=http -ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/docker_images +ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/docker_images ReleaseHost="cloud21.cvtt.vpn" diff --git a/cronjobs/save_cronjobs.sh b/cronjobs/save_cronjobs.sh index fabc2b5..227de99 100755 --- a/cronjobs/save_cronjobs.sh +++ b/cronjobs/save_cronjobs.sh @@ -9,7 +9,7 @@ get_user_hosts() { local User=${1} local Domain=${2} - Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts" + Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts" Cmd="${Cmd} | ${HOME}/bin/hjson -j" Cmd="${Cmd} | jq -r" Cmd="${Cmd} --arg domain \"${Domain}\"" diff --git a/docker/market_data/hist/__DEPRECATED__/alpaca_md_day.sh b/docker/market_data/hist/__DEPRECATED__/alpaca_md_day.sh index 3111f42..a8d369b 100755 --- a/docker/market_data/hist/__DEPRECATED__/alpaca_md_day.sh +++ b/docker/market_data/hist/__DEPRECATED__/alpaca_md_day.sh @@ -17,7 +17,7 @@ is_container_running() { fi } -export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS +export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS is_business_day() { dt=${1} diff --git a/docker/market_data/hist/crypto_md_day.sh b/docker/market_data/hist/crypto_md_day.sh index c9020f3..d4f5947 100755 --- a/docker/market_data/hist/crypto_md_day.sh +++ b/docker/market_data/hist/crypto_md_day.sh @@ -16,7 +16,7 @@ Cmd+=" --network=host" Cmd+=" --name=crypto_md_day.${host}.${date}" Cmd+=" --rm" Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" ${DockerImage}" Cmd+=" -h ${host}" Cmd+=" -d ${date}" diff --git a/docker/market_data/hist/load_alpaca_hbar.sh b/docker/market_data/hist/load_alpaca_hbar.sh index 3ff9d8b..9cf3f69 100755 --- a/docker/market_data/hist/load_alpaca_hbar.sh +++ b/docker/market_data/hist/load_alpaca_hbar.sh @@ -15,7 +15,7 @@ is_container_running() { fi } -export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS +export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS is_business_day() { dt=${1} @@ -92,7 +92,7 @@ Cmd+=" --name=${ContainerName}" Cmd+=" --rm" Cmd+=" --volume=${LogDir}:/logs" Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" ${DockerImage}" Cmd+=" -d ${date_to_load}" if [ -n "${NumSymbols}" ]; then diff --git a/docker/market_data/hist/load_alpaca_qat.sh b/docker/market_data/hist/load_alpaca_qat.sh index afbacfa..065b22a 100755 --- a/docker/market_data/hist/load_alpaca_qat.sh +++ b/docker/market_data/hist/load_alpaca_qat.sh @@ -15,7 +15,7 @@ is_container_running() { fi } -export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS +export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS is_business_day() { dt=${1} @@ -97,7 +97,7 @@ Cmd+=" --name=${ContainerName}" Cmd+=" --rm" Cmd+=" --volume=${LogDir}:/logs" Cmd+=" --volume=${HOME}/.creds:/.creds" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" ${DockerImage}" Cmd+=" -d ${date_to_load}" Cmd+=" -S ${Symbols}" diff --git a/docker/market_data/recorder/md_recorder.sh b/docker/market_data/recorder/md_recorder.sh index 056b70d..2959517 100755 --- a/docker/market_data/recorder/md_recorder.sh +++ b/docker/market_data/recorder/md_recorder.sh @@ -52,7 +52,7 @@ Cmd+=" --name=${ContainerName}" Cmd+=" -v /home/cvtt/.creds:/.creds" Cmd+=" -v /home/cvtt/prod/data:/app/data" Cmd+=" -v /home/cvtt/prod/logs:/logs" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" cloud21.cvtt.vpn:5500/md_recorder:latest" Cmd+=" ${job}" diff --git a/docker/market_data/recorder/md_recorder_monitor.sh b/docker/market_data/recorder/md_recorder_monitor.sh index efca24c..59b93d7 100755 --- a/docker/market_data/recorder/md_recorder_monitor.sh +++ b/docker/market_data/recorder/md_recorder_monitor.sh @@ -52,7 +52,7 @@ Cmd+=" --pull=always" Cmd+=" --name=${ContainerName}" Cmd+=" -v /home/cvtt/.creds:/.creds" Cmd+=" -v /home/cvtt/prod/logs:/logs" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" cloud21.cvtt.vpn:5500/md_recorder_monitor:latest" Cmd+=" ${Instance} ${AdminPort}" diff --git a/docker/research/crypto_exch_stats.sh b/docker/research/crypto_exch_stats.sh index f1efc4b..34eeb3f 100755 --- a/docker/research/crypto_exch_stats.sh +++ b/docker/research/crypto_exch_stats.sh @@ -8,7 +8,7 @@ Cmd+=" --network=host" Cmd+=" --name=crypto_exch_stats" Cmd+=" --volume=${HOME}/prod/data:/app/data" Cmd+=" --volume=${HOME}/prod/logs:/logs" -Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789" +Cmd+=" -e CONFIG_SERVICE=cloud16.cvtt.vpn:6789" Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest" echo ${Cmd} diff --git a/research/prepare_eqt_simdata.sh b/research/prepare_eqt_simdata.sh index 945f13b..47d93ba 100755 --- a/research/prepare_eqt_simdata.sh +++ b/research/prepare_eqt_simdata.sh @@ -15,7 +15,7 @@ is_business_day() { dt=${1} date=$(date -d "${dt}" +"%Y-%m-%d") - CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS + CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS URL="${CalendarURL}&start=${date}&end=${date}" open_time=$(curl -s "${URL}" | jq '.[] | .open_time') if [ -n "${open_time}" ]; then diff --git a/scripts/__DEPRECATED__/archive_yesterday_md.sh b/scripts/__DEPRECATED__/archive_yesterday_md.sh index 442b820..1b93651 100755 --- a/scripts/__DEPRECATED__/archive_yesterday_md.sh +++ b/scripts/__DEPRECATED__/archive_yesterday_md.sh @@ -31,7 +31,7 @@ echo "Schemas=${Schemas}" Cmd= Cmd="${Python}" Cmd="${Cmd} ${RootDir}/cvttpy/research/utils/archive_ts_md.py" -Cmd="${Cmd} --config=http://cloud23.cvtt.vpn:6789/apps/md_recorder" +Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder" Cmd="${Cmd} --db_credentials_key=${CredKey}" Cmd="${Cmd} --date=${yesterday}" Cmd="${Cmd} --schemas=${Schemas}" diff --git a/scripts/healthcheck/cvtt_ts_status_check.sh b/scripts/healthcheck/cvtt_ts_status_check.sh index 897e58e..3cb9a2d 100755 --- a/scripts/healthcheck/cvtt_ts_status_check.sh +++ b/scripts/healthcheck/cvtt_ts_status_check.sh @@ -3,7 +3,7 @@ usage() { echo -n "Usage: ${0}" echo -n " [-c ]" - echo -n " [-s ]" + echo -n " [-s ]" echo -n " [-N ]" echo exit 1 @@ -23,7 +23,7 @@ export PYTHONPATH=${RootDir} StatusChannel=Status-CVTT AlertChannel=Alerts-CVTT Sender=${RootDir}/ops/utils/send_mmost.sh -ConfigService=http://cloud23.cvtt.vpn:6789 +ConfigService=http://cloud16.cvtt.vpn:6789 Config=apps/cvtt_eqt_alpaca Name="ALPACA-SNDBX" diff --git a/scripts/healthcheck/hosts_check.sh b/scripts/healthcheck/hosts_check.sh index 16b47f2..adf9324 100755 --- a/scripts/healthcheck/hosts_check.sh +++ b/scripts/healthcheck/hosts_check.sh @@ -13,7 +13,7 @@ RootDir="${HOME}/prod" AlertChannel=Alerts-CVTT Sender=${RootDir}/ops/utils/send_mmost.sh -ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts +ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j) diff --git a/scripts/healthcheck/services_check.sh b/scripts/healthcheck/services_check.sh index 4fd75fc..30e73ea 100755 --- a/scripts/healthcheck/services_check.sh +++ b/scripts/healthcheck/services_check.sh @@ -12,7 +12,7 @@ RootDir="${HOME}/prod" AlertChannel=Alerts-CVTT Sender=${RootDir}/ops/utils/send_mmost.sh -ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/cvtt_services +ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_services SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j) diff --git a/scripts/healthcheck/storage_health_check.sh b/scripts/healthcheck/storage_health_check.sh index 8077884..8176bb1 100755 --- a/scripts/healthcheck/storage_health_check.sh +++ b/scripts/healthcheck/storage_health_check.sh @@ -18,7 +18,7 @@ get_user_hosts() { local User=${1} local Domain=${2} - Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts" + Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts" Cmd+=" | ${HOME}/bin/hjson -j" Cmd+=" | jq -r" Cmd+=" --arg domain \"${Domain}\"" diff --git a/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh b/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh index c6da9f1..4579e52 100755 --- a/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh +++ b/scripts/mkt_data/alpaca_hist_md/alpaca_md.sh @@ -4,7 +4,7 @@ Start=${1} NumJobs=${2} InstListFile=${3} -export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS +export CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS is_business_day() { dt=${1} @@ -46,7 +46,7 @@ echo "Start=${Start} End=${End} NumJobs=${NumJobs}" export PYTHONPATH=/home/cvtt/prod export Python=/home/cvtt/.pyenv/python3.12-venv/bin/python3 -export Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md +export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py export OutputDir=/home/cvtt/prod/archive/md_archive/equity/alpaca_md # Local diff --git a/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh b/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh index 5f9abd4..70125f4 100755 --- a/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh +++ b/scripts/mkt_data/alpaca_hist_md/load_rel_liq.sh @@ -5,7 +5,7 @@ export PYTHONPATH=${HOME}/prod Python=${HOME}/.pyenv/python3.12-venv/bin/python3 Script=${HOME}/prod/cvttpy/exchanges/alpaca/hist_md/rl_calc_loader.py DbFile=${HOME}/prod/data/rel_liquidity.db -Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md_eqt +Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md_eqt Cmd="${Python}" Cmd="${Cmd} ${Script}" diff --git a/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh b/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh index e58ea1c..e8f3aba 100755 --- a/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh +++ b/scripts/mkt_data/alpaca_hist_md/run_alpaca_hist_md.sh @@ -3,7 +3,7 @@ is_business_day() { dt=${1} - CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS + CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') if [ -n "${open_time}" ]; then return 0 diff --git a/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh b/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh index 21de54c..6670f2f 100755 --- a/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh +++ b/scripts/mkt_data/crypto_hist_md/exchanges_trading_stats.sh @@ -4,7 +4,7 @@ export PYTHONPATH=/home/cvtt/prod Cmd="/home/cvtt/.pyenv/python3.12-venv/bin/python3" Cmd="${Cmd} /home/cvtt/prod/cvttpy/apps/research/exchange_trading_stats.py" -Cmd="${Cmd} --config=http://cloud23.cvtt.vpn:6789/apps/tests/listen_market_data" +Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/tests/listen_market_data" Cmd="${Cmd} --active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN" Cmd="${Cmd} --instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD" Cmd="${Cmd} --db_file=/home/cvtt/prod/data/exchange_trading_stats.db"